##// END OF EJS Templates
schain-jc
RSS
merge 2.3
José Chávez -
r1125:ba0b6ae662be merge
parent child
Show More
Side by Side Unified
Context file:
r1125:ba0b6ae662be Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,26
1 FROM python:2.7-slim
2
3 RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommends \
4 git \
5 gcc \
6 libpng-dev \
7 libfreetype6-dev \
8 libopenblas-dev \
9 liblapack-dev \
10 libatlas-base-dev \
11 libssl-dev \
12 libhdf5-dev \
13 && git clone --branch v2.3 --depth 1 \
14 http://jro-dev.igp.gob.pe/rhodecode/schain \
15 && pip install numpy \
16 && cd schain \
17 && pip install . \
18 && rm -rf * \
19 && apt-get purge -y --auto-remove git gcc \
20 && rm -rf /var/lib/apt/lists/*
21
22 ENV BACKEND="Agg"
23
24 VOLUME /data
25
26 ENTRYPOINT ["schain"]
Show file before | Show file after | Hide comments
@@ -1,153 +1,155
1 1 # Signal Chain
2 2
3 3 ## Introduction
4 4
5 5 Signal Chain (SCh) is a radar data processing library developed using [Python](www.python.org) at JRO. SCh provides modules to read, write, process and plot data.
6 6
7 7 ## Installation
8 8
9 Install system dependencies, clone the latest version from [git](http://jro-dev.igp.gob.pe/rhodecode/schain/) and install it as a normal python package.
9 Install system dependencies, clone the latest version from [here](http://jro-dev.igp.gob.pe/rhodecode/schain/) and install it as a normal python package.
10 10
11 11 ### Linux based system
12 12 ```
13 13 $ sudo apt-get install python-pip python-dev gfortran libpng-dev freetype* libblas-dev liblapack-dev libatlas-base-dev python-qt4 python-tk libssl-dev libhdf5-dev
14 14 $ git clone http://jro-dev.igp.gob.pe/rhodecode/schain/
15 15 $ cd schain
16 16 $ sudo pip install ./
17 17
18 18 ```
19 **It is recommended to install schain in a virtual environment**
20 ```
21 $ sudo pip install virtualenv
22 $ virtualenv /path/to/virtual --system-site-packages
23 $ source /path/to/virtual/bin/activate
24 (virtual) $ cd schain
25 (virtual) $ pip install ./
26
27 ```
28 19
29 20 ### MAC Os
30 21 ```
22 $ brew install python
31 23 $ brew install cartr/qt4/pyqt
32 24 $ git clone http://jro-dev.igp.gob.pe/rhodecode/schain/
33 25 $ cd schain
34 26 $ pip install ./
35 27 ```
36 28
37 if ```pip install ./``` does not work, install a proper python enviroment, and repeat the steps.
29 **It is recommended to install schain in a virtual environment**
38 30 ```
39 $ brew install python
31 $ virtualenv /path/to/virtual
32 $ source /path/to/virtual/bin/activate
33 (virtual) $ cd schain
34 (virtual) $ pip install ./
35 (virtual) $ bash link_PyQt4.sh
40 36 ```
41 37
42 ### GUI Installation
38 ### Docker
39
40 Download Dockerfile from the repository, and create a docker image
43 41
44 42 ```
45 $ sudo apt-get install python-pip python-dev gfortran libpng-dev freetype* libblas-dev liblapack-dev libatlas-base-dev python-qt4 python-tk libssl-dev libhdf5-dev
46 $ (virtual) bash link_PyQt4.sh
43 $ docker build -t schain .
47 44 ```
48 45
46 You can run a container using an xml file or a schain script also you need to mount a volume for the data input and for the output files/plots
47 ```
48 $ docker run -it --rm --volume /path/to/host/data:/data schain xml /data/test.xml
49 $ docker run -it --rm --volume /path/to/host/data:/data --entrypoint=/bin/python schain /data/test.py
50 ```
49 51
50 52 ## First Script
51 53
52 54 Read Spectra data (.pdata) - remove dc - plot spectra & RTI
53 55
54 56 Import SCh and creating a project
55 57
56 58 ```python
57 59 #!/usr/bin/python
58 60
59 61 from schainpy.controller import Project
60 62
61 63 controller = Project()
62 64 controller.setup(id = '100',
63 65 name='test',
64 66 description='Basic experiment')
65 67
66 68
67 69 ```
68 70
69 71 Adding read unit and operations
70 72
71 73 ```python
72 74 read_unit = controller.addReadUnit(datatype='Spectra',
73 75 path='/path/to/pdata/',
74 76 startDate='2014/01/31',
75 77 endDate='2014/03/31',
76 78 startTime='00:00:00',
77 79 endTime='23:59:59',
78 80 online=0,
79 81 walk=0)
80 82
81 83 proc_unit = controller.addProcUnit(datatype='Spectra',
82 84 inputId=read_unit.getId())
83 85
84 86 op = proc_unit.addOperation(name='selectChannels')
85 87 op.addParameter(name='channelList', value='0,1', format='intlist')
86 88
87 89 op = proc_unit.addOperation(name='selectHeights')
88 90 op.addParameter(name='minHei', value='80', format='float')
89 91 op.addParameter(name='maxHei', value='200', format='float')
90 92
91 93 op = proc_unit.addOperation(name='removeDC')
92 94
93 95 ```
94 96
95 97 Plotting data & start project
96 98
97 99 ```python
98 100 op = proc_unit.addOperation(name='SpectraPlot', optype='other')
99 101 op.addParameter(name='id', value='1', format='int')
100 102 op.addParameter(name='wintitle', value='Spectra', format='str')
101 103
102 104 op = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
103 105 op.addParameter(name='id', value='2', format='int')
104 106 op.addParameter(name='wintitle', value='RTI', format='str')
105 107
106 108 controller.start()
107 109
108 110 ```
109 111
110 112 Full script
111 113
112 114
113 115 ```python
114 116 #!/usr/bin/python
115 117
116 118 from schainpy.controller import Project
117 119
118 120 controller = Project()
119 121 controller.setup(id = '100',
120 122 name='test',
121 123 description='Basic experiment')
122 124 read_unit = controller.addReadUnit(datatype='Spectra',
123 125 path='/path/to/pdata/',
124 126 startDate='2014/01/31',
125 127 endDate='2014/03/31',
126 128 startTime='00:00:00',
127 129 endTime='23:59:59',
128 130 online=0,
129 131 walk=0)
130 132
131 133 proc_unit = controller.addProcUnit(datatype='Spectra',
132 134 inputId=read_unit.getId())
133 135
134 136 op = proc_unit.addOperation(name='selectChannels')
135 137 op.addParameter(name='channelList', value='0,1', format='intlist')
136 138
137 139 op = proc_unit.addOperation(name='selectHeights')
138 140 op.addParameter(name='minHei', value='80', format='float')
139 141 op.addParameter(name='maxHei', value='200', format='float')
140 142
141 143 op = proc_unit.addOperation(name='removeDC')
142 144
143 145 op = proc_unit.addOperation(name='SpectraPlot', optype='other')
144 146 op.addParameter(name='id', value='6', format='int')
145 147 op.addParameter(name='wintitle', value='Spectra', format='str')
146 148
147 149 op = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
148 150 op.addParameter(name='id', value='2', format='int')
149 151 op.addParameter(name='wintitle', value='RTI', format='str')
150 152
151 153 controller.start()
152 154
153 155 ``` No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,17 +1,15
1 1 #!/bin/bash
2 2 # This hook is run after a new virtualenv is activated.
3 # ~/.virtualenvs/postmkvirtualenv
4
5 libs=( PyQt4 sip.so )
6 3
7 4 python_version=python$(python -c "import sys; print (str(sys.version_info[0])+'.'+str(sys.version_info[1]))")
8 5 var=( $(which -a $python_version) )
9 6
10 7 get_python_lib_cmd="from distutils.sysconfig import get_python_lib; print (get_python_lib())"
11 8 lib_virtualenv_path=$(python -c "$get_python_lib_cmd")
12 9 lib_system_path=$(${var[-1]} -c "$get_python_lib_cmd")
10 sip_path=$(ls $lib_system_path/sip*.so)
13 11
14 for lib in ${libs[@]}
15 do
16 ln -s $lib_system_path/$lib $lib_virtualenv_path/$lib
17 done
12 echo "Linking Qt4..."
13 ln -s $lib_system_path/PyQt4 $lib_virtualenv_path/PyQt4
14 echo "Linking SIP..."
15 ln -s $sip_path $lib_virtualenv_path/sip.so
Show file before | Show file after | Hide comments
1 NO CONTENT: file renamed from schaincli/README.md to schainpy/cli/README.md
Show file before | Show file after | Hide comments
1 NO CONTENT: file renamed from schaincli/__init__.py to schainpy/cli/__init__.py
Show file before | Show file after | Hide comments
@@ -1,168 +1,168
1 1 import click
2 2 import schainpy
3 3 import subprocess
4 4 import os
5 5 import sys
6 6 import glob
7 7 save_stdout = sys.stdout
8 8 sys.stdout = open('trash', 'w')
9 9 from multiprocessing import cpu_count
10 from schaincli import templates
11 10 from schainpy.controller import Project
12 11 from schainpy.model import Operation, ProcessingUnit
13 12 from schainpy.utils import log
14 13 from importlib import import_module
15 14 from pydoc import locate
16 15 from fuzzywuzzy import process
17 16 from schainpy.utils import paramsFinder
17 import templates
18 18 sys.stdout = save_stdout
19 19
20 20
21 21 def print_version(ctx, param, value):
22 22 if not value or ctx.resilient_parsing:
23 23 return
24 24 click.echo(schainpy.__version__)
25 25 ctx.exit()
26 26
27 27
28 28 cliLogger = log.makelogger('schain cli')
29 29 PREFIX = 'experiment'
30 30
31 31
32 32 @click.command()
33 33 @click.option('--version', '-v', is_flag=True, callback=print_version, help='SChain version', type=str)
34 34 @click.argument('command', default='run', required=True)
35 35 @click.argument('nextcommand', default=None, required=False, type=str)
36 36 def main(command, nextcommand, version):
37 37 """COMMAND LINE INTERFACE FOR SIGNAL CHAIN - JICAMARCA RADIO OBSERVATORY \n
38 38 Available commands.\n
39 39 --xml: runs a schain XML generated file\n
40 40 run: runs any python script starting 'experiment_'\n
41 41 generate: generates a template schain script\n
42 42 search: return avilable operations, procs or arguments of the give operation/proc\n"""
43 43 if command == 'xml':
44 44 runFromXML(nextcommand)
45 45 elif command == 'generate':
46 46 generate()
47 47 elif command == 'test':
48 48 test()
49 49 elif command == 'run':
50 50 runschain(nextcommand)
51 51 elif command == 'search':
52 52 search(nextcommand)
53 53 else:
54 54 log.error('Command {} is not defined'.format(command))
55 55
56 56
57 57 def check_module(possible, instance):
58 58 def check(x):
59 59 try:
60 60 instancia = locate('schainpy.model.{}'.format(x))
61 61 return isinstance(instancia(), instance)
62 62 except Exception as e:
63 63 return False
64 64 clean = clean_modules(possible)
65 65 return [x for x in clean if check(x)]
66 66
67 67
68 68 def clean_modules(module):
69 69 noEndsUnder = [x for x in module if not x.endswith('__')]
70 70 noStartUnder = [x for x in noEndsUnder if not x.startswith('__')]
71 71 noFullUpper = [x for x in noStartUnder if not x.isupper()]
72 72 return noFullUpper
73 73
74 74
75 75 def search(nextcommand):
76 76 if nextcommand is None:
77 77 log.error('There is no Operation/ProcessingUnit to search')
78 78 elif nextcommand == 'procs':
79 79 procs = paramsFinder.getProcs()
80 80 log.success(
81 81 'Current ProcessingUnits are:\n\033[1m{}\033[0m'.format('\n'.join(procs)))
82 82
83 83 elif nextcommand == 'operations':
84 84 operations = paramsFinder.getOperations()
85 85 log.success('Current Operations are:\n\033[1m{}\033[0m'.format(
86 86 '\n'.join(operations)))
87 87 else:
88 88 try:
89 89 args = paramsFinder.getArgs(nextcommand)
90 90 log.warning(
91 91 'Use this feature with caution. It may not return all the allowed arguments')
92 92 if len(args) == 0:
93 93 log.success('{} has no arguments'.format(nextcommand))
94 94 else:
95 95 log.success('Showing {} arguments:\n\033[1m{}\033[0m'.format(
96 96 nextcommand, '\n'.join(args)))
97 97 except Exception as e:
98 98 log.error('Module {} does not exists'.format(nextcommand))
99 99 allModules = paramsFinder.getAll()
100 100 similar = process.extractOne(nextcommand, allModules)[0]
101 101 log.success('Showing {} instead'.format(similar))
102 102 search(similar)
103 103
104 104
105 105 def runschain(nextcommand):
106 106 if nextcommand is None:
107 107 currentfiles = glob.glob('./{}_*.py'.format(PREFIX))
108 108 numberfiles = len(currentfiles)
109 109 if numberfiles > 1:
110 110 log.error('There is more than one file to run')
111 111 elif numberfiles == 1:
112 112 subprocess.call(['python ' + currentfiles[0]], shell=True)
113 113 else:
114 114 log.error('There is no file to run')
115 115 else:
116 116 try:
117 117 subprocess.call(['python ' + nextcommand], shell=True)
118 118 except Exception as e:
119 119 log.error("I cannot run the file. Does it exists?")
120 120
121 121
122 122 def basicInputs():
123 123 inputs = {}
124 124 inputs['desc'] = click.prompt(
125 125 'Enter a description', default="A schain project", type=str)
126 126 inputs['name'] = click.prompt(
127 127 'Name of the project', default="project", type=str)
128 128 inputs['path'] = click.prompt('Data path', default=os.getcwd(
129 129 ), type=click.Path(exists=True, resolve_path=True))
130 130 inputs['startDate'] = click.prompt(
131 131 'Start date', default='1970/01/01', type=str)
132 132 inputs['endDate'] = click.prompt(
133 133 'End date', default='2017/12/31', type=str)
134 134 inputs['startHour'] = click.prompt(
135 135 'Start hour', default='00:00:00', type=str)
136 136 inputs['endHour'] = click.prompt('End hour', default='23:59:59', type=str)
137 137 inputs['figpath'] = inputs['path'] + '/figs'
138 138 return inputs
139 139
140 140
141 141 def generate():
142 142 inputs = basicInputs()
143 143 inputs['multiprocess'] = click.confirm('Is this a multiprocess script?')
144 144 if inputs['multiprocess']:
145 145 inputs['nProcess'] = click.prompt(
146 146 'How many process?', default=cpu_count(), type=int)
147 147 current = templates.multiprocess.format(**inputs)
148 148 else:
149 149 current = templates.basic.format(**inputs)
150 150 scriptname = '{}_{}.py'.format(PREFIX, inputs['name'])
151 151 script = open(scriptname, 'w')
152 152 try:
153 153 script.write(current)
154 154 log.success('Script {} generated'.format(scriptname))
155 155 except Exception as e:
156 156 log.error('I cannot create the file. Do you have writing permissions?')
157 157
158 158
159 159 def test():
160 160 log.warning('testing')
161 161
162 162
163 163 def runFromXML(filename):
164 164 controller = Project()
165 165 if not controller.readXml(filename):
166 166 return
167 167 controller.start()
168 168 return
Show file before | Show file after | Hide comments
1 NO CONTENT: file renamed from schaincli/templates.py to schainpy/cli/templates.py
Show file before | Show file after | Hide comments
1 NO CONTENT: file renamed from schaincli/tests/__init__.py to schainpy/cli/tests/__init__.py
Show file before | Show file after | Hide comments
@@ -1,29 +1,29
1 1 import pytest
2 2 from click.testing import CliRunner
3 from schaincli import cli
3 from schainpy.cli import cli
4 4
5 5
6 6 @pytest.fixture
7 7 def runner():
8 8 return CliRunner()
9 9
10 10
11 11 def test_cli(runner):
12 12 result = runner.invoke(cli.main)
13 13 assert result.exit_code == 0
14 14 assert not result.exception
15 15 assert result.output.strip() == 'Hello, world.'
16 16
17 17
18 18 def test_cli_with_option(runner):
19 19 result = runner.invoke(cli.main, ['--as-cowboy'])
20 20 assert not result.exception
21 21 assert result.exit_code == 0
22 22 assert result.output.strip() == 'Howdy, world.'
23 23
24 24
25 25 def test_cli_with_arg(runner):
26 26 result = runner.invoke(cli.main, ['Jicamarca'])
27 27 assert result.exit_code == 0
28 28 assert not result.exception
29 29 assert result.output.strip() == 'Hello, Jicamarca.'
Show file before | Show file after | Hide comments
@@ -1,1317 +1,1323
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5
6 6 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 10 import math
11 11 import time
12 12 from multiprocessing import Process, cpu_count
13 13
14 14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
15 15 from xml.dom import minidom
16 16
17 17 import schainpy
18 18 import schainpy.admin
19 19 from schainpy.model import *
20 20 from schainpy.utils import log
21 21
22 22 DTYPES = {
23 23 'Voltage': '.r',
24 24 'Spectra': '.pdata'
25 25 }
26 26
27 27
28 28 def MPProject(project, n=cpu_count()):
29 29 '''
30 30 Project wrapper to run schain in n processes
31 31 '''
32 32
33 33 rconf = project.getReadUnitObj()
34 34 op = rconf.getOperationObj('run')
35 35 dt1 = op.getParameterValue('startDate')
36 36 dt2 = op.getParameterValue('endDate')
37 tm1 = op.getParameterValue('startTime')
38 tm2 = op.getParameterValue('endTime')
37 39 days = (dt2 - dt1).days
38 40
39 41 for day in range(days + 1):
40 42 skip = 0
41 43 cursor = 0
42 44 processes = []
43 45 dt = dt1 + datetime.timedelta(day)
44 46 dt_str = dt.strftime('%Y/%m/%d')
45 47 reader = JRODataReader()
46 48 paths, files = reader.searchFilesOffLine(path=rconf.path,
47 49 startDate=dt,
48 50 endDate=dt,
51 startTime=tm1,
52 endTime=tm2,
49 53 ext=DTYPES[rconf.datatype])
50 54 nFiles = len(files)
51 55 if nFiles == 0:
52 56 continue
53 skip = int(math.ceil(nFiles / n))
57 skip = int(math.ceil(nFiles / n))
54 58 while nFiles > cursor * skip:
55 59 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
56 60 skip=skip)
57 61 p = project.clone()
58 62 p.start()
59 63 processes.append(p)
60 64 cursor += 1
61 65
62 66 def beforeExit(exctype, value, trace):
63 67 for process in processes:
64 68 process.terminate()
65 69 process.join()
66 70 print traceback.print_tb(trace)
67 71
68 72 sys.excepthook = beforeExit
69 73
70 74 for process in processes:
71 75 process.join()
72 76 process.terminate()
73 77
74 78 time.sleep(3)
75 79
76 80
77 81 class ParameterConf():
78 82
79 83 id = None
80 84 name = None
81 85 value = None
82 86 format = None
83 87
84 88 __formated_value = None
85 89
86 90 ELEMENTNAME = 'Parameter'
87 91
88 92 def __init__(self):
89 93
90 94 self.format = 'str'
91 95
92 96 def getElementName(self):
93 97
94 98 return self.ELEMENTNAME
95 99
96 100 def getValue(self):
97 101
98 102 value = self.value
99 103 format = self.format
100 104
101 105 if self.__formated_value != None:
102 106
103 107 return self.__formated_value
104 108
105 109 if format == 'obj':
106 110 return value
107 111
108 112 if format == 'str':
109 113 self.__formated_value = str(value)
110 114 return self.__formated_value
111 115
112 116 if value == '':
113 117 raise ValueError, '%s: This parameter value is empty' % self.name
114 118
115 119 if format == 'list':
116 120 strList = value.split(',')
117 121
118 122 self.__formated_value = strList
119 123
120 124 return self.__formated_value
121 125
122 126 if format == 'intlist':
123 127 '''
124 128 Example:
125 129 value = (0,1,2)
126 130 '''
127 131
128 132 new_value = ast.literal_eval(value)
129 133
130 134 if type(new_value) not in (tuple, list):
131 135 new_value = [int(new_value)]
132 136
133 137 self.__formated_value = new_value
134 138
135 139 return self.__formated_value
136 140
137 141 if format == 'floatlist':
138 142 '''
139 143 Example:
140 144 value = (0.5, 1.4, 2.7)
141 145 '''
142 146
143 147 new_value = ast.literal_eval(value)
144 148
145 149 if type(new_value) not in (tuple, list):
146 150 new_value = [float(new_value)]
147 151
148 152 self.__formated_value = new_value
149 153
150 154 return self.__formated_value
151 155
152 156 if format == 'date':
153 157 strList = value.split('/')
154 158 intList = [int(x) for x in strList]
155 159 date = datetime.date(intList[0], intList[1], intList[2])
156 160
157 161 self.__formated_value = date
158 162
159 163 return self.__formated_value
160 164
161 165 if format == 'time':
162 166 strList = value.split(':')
163 167 intList = [int(x) for x in strList]
164 168 time = datetime.time(intList[0], intList[1], intList[2])
165 169
166 170 self.__formated_value = time
167 171
168 172 return self.__formated_value
169 173
170 174 if format == 'pairslist':
171 175 '''
172 176 Example:
173 177 value = (0,1),(1,2)
174 178 '''
175 179
176 180 new_value = ast.literal_eval(value)
177 181
178 182 if type(new_value) not in (tuple, list):
179 183 raise ValueError, '%s has to be a tuple or list of pairs' % value
180 184
181 185 if type(new_value[0]) not in (tuple, list):
182 186 if len(new_value) != 2:
183 187 raise ValueError, '%s has to be a tuple or list of pairs' % value
184 188 new_value = [new_value]
185 189
186 190 for thisPair in new_value:
187 191 if len(thisPair) != 2:
188 192 raise ValueError, '%s has to be a tuple or list of pairs' % value
189 193
190 194 self.__formated_value = new_value
191 195
192 196 return self.__formated_value
193 197
194 198 if format == 'multilist':
195 199 '''
196 200 Example:
197 201 value = (0,1,2),(3,4,5)
198 202 '''
199 203 multiList = ast.literal_eval(value)
200 204
201 205 if type(multiList[0]) == int:
202 206 multiList = ast.literal_eval('(' + value + ')')
203 207
204 208 self.__formated_value = multiList
205 209
206 210 return self.__formated_value
207 211
208 212 if format == 'bool':
209 213 value = int(value)
210 214
211 215 if format == 'int':
212 216 value = float(value)
213 217
214 218 format_func = eval(format)
215 219
216 220 self.__formated_value = format_func(value)
217 221
218 222 return self.__formated_value
219 223
220 224 def updateId(self, new_id):
221 225
222 226 self.id = str(new_id)
223 227
224 228 def setup(self, id, name, value, format='str'):
225 229 self.id = str(id)
226 230 self.name = name
227 231 if format == 'obj':
228 232 self.value = value
229 233 else:
230 234 self.value = str(value)
231 235 self.format = str.lower(format)
232 236
233 237 self.getValue()
234 238
235 239 return 1
236 240
237 241 def update(self, name, value, format='str'):
238 242
239 243 self.name = name
240 244 self.value = str(value)
241 245 self.format = format
242 246
243 247 def makeXml(self, opElement):
244 248 if self.name not in ('queue',):
245 249 parmElement = SubElement(opElement, self.ELEMENTNAME)
246 250 parmElement.set('id', str(self.id))
247 251 parmElement.set('name', self.name)
248 252 parmElement.set('value', self.value)
249 253 parmElement.set('format', self.format)
250 254
251 255 def readXml(self, parmElement):
252 256
253 257 self.id = parmElement.get('id')
254 258 self.name = parmElement.get('name')
255 259 self.value = parmElement.get('value')
256 260 self.format = str.lower(parmElement.get('format'))
257 261
258 262 # Compatible with old signal chain version
259 263 if self.format == 'int' and self.name == 'idfigure':
260 264 self.name = 'id'
261 265
262 266 def printattr(self):
263 267
264 268 print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
265 269
266 270
267 271 class OperationConf():
268 272
269 273 id = None
270 274 name = None
271 275 priority = None
272 276 type = None
273 277
274 278 parmConfObjList = []
275 279
276 280 ELEMENTNAME = 'Operation'
277 281
278 282 def __init__(self):
279 283
280 284 self.id = '0'
281 285 self.name = None
282 286 self.priority = None
283 287 self.type = 'self'
284 288
285 289 def __getNewId(self):
286 290
287 291 return int(self.id) * 10 + len(self.parmConfObjList) + 1
288 292
289 293 def updateId(self, new_id):
290 294
291 295 self.id = str(new_id)
292 296
293 297 n = 1
294 298 for parmObj in self.parmConfObjList:
295 299
296 300 idParm = str(int(new_id) * 10 + n)
297 301 parmObj.updateId(idParm)
298 302
299 303 n += 1
300 304
301 305 def getElementName(self):
302 306
303 307 return self.ELEMENTNAME
304 308
305 309 def getParameterObjList(self):
306 310
307 311 return self.parmConfObjList
308 312
309 313 def getParameterObj(self, parameterName):
310 314
311 315 for parmConfObj in self.parmConfObjList:
312 316
313 317 if parmConfObj.name != parameterName:
314 318 continue
315 319
316 320 return parmConfObj
317 321
318 322 return None
319 323
320 324 def getParameterObjfromValue(self, parameterValue):
321 325
322 326 for parmConfObj in self.parmConfObjList:
323 327
324 328 if parmConfObj.getValue() != parameterValue:
325 329 continue
326 330
327 331 return parmConfObj.getValue()
328 332
329 333 return None
330 334
331 335 def getParameterValue(self, parameterName):
332 336
333 337 parameterObj = self.getParameterObj(parameterName)
334 338
335 339 # if not parameterObj:
336 340 # return None
337 341
338 342 value = parameterObj.getValue()
339 343
340 344 return value
341 345
342 346 def getKwargs(self):
343 347
344 348 kwargs = {}
345 349
346 350 for parmConfObj in self.parmConfObjList:
347 351 if self.name == 'run' and parmConfObj.name == 'datatype':
348 352 continue
349 353
350 354 kwargs[parmConfObj.name] = parmConfObj.getValue()
351 355
352 356 return kwargs
353 357
354 358 def setup(self, id, name, priority, type):
355 359
356 360 self.id = str(id)
357 361 self.name = name
358 362 self.type = type
359 363 self.priority = priority
360 364
361 365 self.parmConfObjList = []
362 366
363 367 def removeParameters(self):
364 368
365 369 for obj in self.parmConfObjList:
366 370 del obj
367 371
368 372 self.parmConfObjList = []
369 373
370 374 def addParameter(self, name, value, format='str'):
371 375
372 376 if value is None:
373 377 return None
374 378 id = self.__getNewId()
375 379
376 380 parmConfObj = ParameterConf()
377 381 if not parmConfObj.setup(id, name, value, format):
378 382 return None
379 383
380 384 self.parmConfObjList.append(parmConfObj)
381 385
382 386 return parmConfObj
383 387
384 388 def changeParameter(self, name, value, format='str'):
385 389
386 390 parmConfObj = self.getParameterObj(name)
387 391 parmConfObj.update(name, value, format)
388 392
389 393 return parmConfObj
390 394
391 395 def makeXml(self, procUnitElement):
392 396
393 397 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
394 398 opElement.set('id', str(self.id))
395 399 opElement.set('name', self.name)
396 400 opElement.set('type', self.type)
397 401 opElement.set('priority', str(self.priority))
398 402
399 403 for parmConfObj in self.parmConfObjList:
400 404 parmConfObj.makeXml(opElement)
401 405
402 406 def readXml(self, opElement):
403 407
404 408 self.id = opElement.get('id')
405 409 self.name = opElement.get('name')
406 410 self.type = opElement.get('type')
407 411 self.priority = opElement.get('priority')
408 412
409 413 # Compatible with old signal chain version
410 414 # Use of 'run' method instead 'init'
411 415 if self.type == 'self' and self.name == 'init':
412 416 self.name = 'run'
413 417
414 418 self.parmConfObjList = []
415 419
416 420 parmElementList = opElement.iter(ParameterConf().getElementName())
417 421
418 422 for parmElement in parmElementList:
419 423 parmConfObj = ParameterConf()
420 424 parmConfObj.readXml(parmElement)
421 425
422 426 # Compatible with old signal chain version
423 427 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
424 428 if self.type != 'self' and self.name == 'Plot':
425 429 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
426 430 self.name = parmConfObj.value
427 431 continue
428 432
429 433 self.parmConfObjList.append(parmConfObj)
430 434
431 435 def printattr(self):
432 436
433 437 print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
434 438 self.id,
435 439 self.name,
436 440 self.type,
437 441 self.priority)
438 442
439 443 for parmConfObj in self.parmConfObjList:
440 444 parmConfObj.printattr()
441 445
442 446 def createObject(self, plotter_queue=None):
443 447
444 448 if self.type == 'self':
445 449 raise ValueError, 'This operation type cannot be created'
446 450
447 451 if self.type == 'plotter':
448 452 if not plotter_queue:
449 453 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
450 454
451 455 opObj = Plotter(self.name, plotter_queue)
452 456
453 457 if self.type == 'external' or self.type == 'other':
454 458
455 459 className = eval(self.name)
456 460 kwargs = self.getKwargs()
457 461
458 462 opObj = className(**kwargs)
459 463
460 464 return opObj
461 465
462 466
463 467 class ProcUnitConf():
464 468
465 469 id = None
466 470 name = None
467 471 datatype = None
468 472 inputId = None
469 473 parentId = None
470 474
471 475 opConfObjList = []
472 476
473 477 procUnitObj = None
474 478 opObjList = []
475 479
476 480 ELEMENTNAME = 'ProcUnit'
477 481
478 482 def __init__(self):
479 483
480 484 self.id = None
481 485 self.datatype = None
482 486 self.name = None
483 487 self.inputId = None
484 488
485 489 self.opConfObjList = []
486 490
487 491 self.procUnitObj = None
488 492 self.opObjDict = {}
489 493
490 494 def __getPriority(self):
491 495
492 496 return len(self.opConfObjList) + 1
493 497
494 498 def __getNewId(self):
495 499
496 500 return int(self.id) * 10 + len(self.opConfObjList) + 1
497 501
498 502 def getElementName(self):
499 503
500 504 return self.ELEMENTNAME
501 505
502 506 def getId(self):
503 507
504 508 return self.id
505 509
506 510 def updateId(self, new_id, parentId=parentId):
507 511
508 512 new_id = int(parentId) * 10 + (int(self.id) % 10)
509 513 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
510 514
511 515 # If this proc unit has not inputs
512 516 if self.inputId == '0':
513 517 new_inputId = 0
514 518
515 519 n = 1
516 520 for opConfObj in self.opConfObjList:
517 521
518 522 idOp = str(int(new_id) * 10 + n)
519 523 opConfObj.updateId(idOp)
520 524
521 525 n += 1
522 526
523 527 self.parentId = str(parentId)
524 528 self.id = str(new_id)
525 529 self.inputId = str(new_inputId)
526 530
527 531 def getInputId(self):
528 532
529 533 return self.inputId
530 534
531 535 def getOperationObjList(self):
532 536
533 537 return self.opConfObjList
534 538
535 539 def getOperationObj(self, name=None):
536 540
537 541 for opConfObj in self.opConfObjList:
538 542
539 543 if opConfObj.name != name:
540 544 continue
541 545
542 546 return opConfObj
543 547
544 548 return None
545 549
546 550 def getOpObjfromParamValue(self, value=None):
547 551
548 552 for opConfObj in self.opConfObjList:
549 553 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
550 554 continue
551 555 return opConfObj
552 556 return None
553 557
554 558 def getProcUnitObj(self):
555 559
556 560 return self.procUnitObj
557 561
558 562 def setup(self, id, name, datatype, inputId, parentId=None):
559 563
560 564 # Compatible with old signal chain version
561 565 if datatype == None and name == None:
562 566 raise ValueError, 'datatype or name should be defined'
563 567
564 568 if name == None:
565 569 if 'Proc' in datatype:
566 570 name = datatype
567 571 else:
568 572 name = '%sProc' % (datatype)
569 573
570 574 if datatype == None:
571 575 datatype = name.replace('Proc', '')
572 576
573 577 self.id = str(id)
574 578 self.name = name
575 579 self.datatype = datatype
576 580 self.inputId = inputId
577 581 self.parentId = parentId
578 582
579 583 self.opConfObjList = []
580 584
581 585 self.addOperation(name='run', optype='self')
582 586
583 587 def removeOperations(self):
584 588
585 589 for obj in self.opConfObjList:
586 590 del obj
587 591
588 592 self.opConfObjList = []
589 593 self.addOperation(name='run')
590 594
591 595 def addParameter(self, **kwargs):
592 596 '''
593 597 Add parameters to 'run' operation
594 598 '''
595 599 opObj = self.opConfObjList[0]
596 600
597 601 opObj.addParameter(**kwargs)
598 602
599 603 return opObj
600 604
601 605 def addOperation(self, name, optype='self'):
602 606
603 607 id = self.__getNewId()
604 608 priority = self.__getPriority()
605 609
606 610 opConfObj = OperationConf()
607 611 opConfObj.setup(id, name=name, priority=priority, type=optype)
608 612
609 613 self.opConfObjList.append(opConfObj)
610 614
611 615 return opConfObj
612 616
613 617 def makeXml(self, projectElement):
614 618
615 619 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
616 620 procUnitElement.set('id', str(self.id))
617 621 procUnitElement.set('name', self.name)
618 622 procUnitElement.set('datatype', self.datatype)
619 623 procUnitElement.set('inputId', str(self.inputId))
620 624
621 625 for opConfObj in self.opConfObjList:
622 626 opConfObj.makeXml(procUnitElement)
623 627
624 628 def readXml(self, upElement):
625 629
626 630 self.id = upElement.get('id')
627 631 self.name = upElement.get('name')
628 632 self.datatype = upElement.get('datatype')
629 633 self.inputId = upElement.get('inputId')
630 634
631 635 if self.ELEMENTNAME == 'ReadUnit':
632 636 self.datatype = self.datatype.replace('Reader', '')
633 637
634 638 if self.ELEMENTNAME == 'ProcUnit':
635 639 self.datatype = self.datatype.replace('Proc', '')
636 640
637 641 if self.inputId == 'None':
638 642 self.inputId = '0'
639 643
640 644 self.opConfObjList = []
641 645
642 646 opElementList = upElement.iter(OperationConf().getElementName())
643 647
644 648 for opElement in opElementList:
645 649 opConfObj = OperationConf()
646 650 opConfObj.readXml(opElement)
647 651 self.opConfObjList.append(opConfObj)
648 652
649 653 def printattr(self):
650 654
651 655 print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
652 656 self.id,
653 657 self.name,
654 658 self.datatype,
655 659 self.inputId)
656 660
657 661 for opConfObj in self.opConfObjList:
658 662 opConfObj.printattr()
659 663
660 664 def getKwargs(self):
661 665
662 666 opObj = self.opConfObjList[0]
663 667 kwargs = opObj.getKwargs()
664 668
665 669 return kwargs
666 670
667 671 def createObjects(self, plotter_queue=None):
668 672
669 673 className = eval(self.name)
670 674 kwargs = self.getKwargs()
671 675 procUnitObj = className(**kwargs)
672 676
673 677 for opConfObj in self.opConfObjList:
674 678
675 679 if opConfObj.type == 'self' and self.name == 'run':
676 680 continue
677 681 elif opConfObj.type == 'self':
678 682 procUnitObj.addOperationKwargs(
679 683 opConfObj.id, **opConfObj.getKwargs())
680 684 continue
681 685
682 686 opObj = opConfObj.createObject(plotter_queue)
683 687
684 688 self.opObjDict[opConfObj.id] = opObj
685 689
686 690 procUnitObj.addOperation(opObj, opConfObj.id)
687 691
688 692 self.procUnitObj = procUnitObj
689 693
690 694 return procUnitObj
691 695
692 696 def run(self):
693 697
694 698 is_ok = False
695 699
696 700 for opConfObj in self.opConfObjList:
697 701
698 702 kwargs = {}
699 703 for parmConfObj in opConfObj.getParameterObjList():
700 704 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
701 705 continue
702 706
703 707 kwargs[parmConfObj.name] = parmConfObj.getValue()
704 708
705 709 sts = self.procUnitObj.call(opType=opConfObj.type,
706 710 opName=opConfObj.name,
707 711 opId=opConfObj.id)
708 712
709 713 is_ok = is_ok or sts
710 714
711 715 return is_ok
712 716
713 717 def close(self):
714 718
715 719 for opConfObj in self.opConfObjList:
716 720 if opConfObj.type == 'self':
717 721 continue
718 722
719 723 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
720 724 opObj.close()
721 725
722 726 self.procUnitObj.close()
723 727
724 728 return
725 729
726 730
727 731 class ReadUnitConf(ProcUnitConf):
728 732
729 733 path = None
730 734 startDate = None
731 735 endDate = None
732 736 startTime = None
733 737 endTime = None
734 738
735 739 ELEMENTNAME = 'ReadUnit'
736 740
737 741 def __init__(self):
738 742
739 743 self.id = None
740 744 self.datatype = None
741 745 self.name = None
742 746 self.inputId = None
743 747
744 748 self.parentId = None
745 749
746 750 self.opConfObjList = []
747 751 self.opObjList = []
748 752
749 753 def getElementName(self):
750 754
751 755 return self.ELEMENTNAME
752 756
753 757 def setup(self, id, name, datatype, path='', startDate='', endDate='',
754 758 startTime='', endTime='', parentId=None, server=None, **kwargs):
755 759
756 760 # Compatible with old signal chain version
757 761 if datatype == None and name == None:
758 762 raise ValueError, 'datatype or name should be defined'
759 763 if name == None:
760 764 if 'Reader' in datatype:
761 765 name = datatype
762 766 datatype = name.replace('Reader','')
763 767 else:
764 768 name = '{}Reader'.format(datatype)
765 769 if datatype == None:
766 770 if 'Reader' in name:
767 771 datatype = name.replace('Reader','')
768 772 else:
769 773 datatype = name
770 774 name = '{}Reader'.format(name)
771 775
772 776 self.id = id
773 777 self.name = name
774 778 self.datatype = datatype
775 779 if path != '':
776 780 self.path = os.path.abspath(path)
777 781 self.startDate = startDate
778 782 self.endDate = endDate
779 783 self.startTime = startTime
780 784 self.endTime = endTime
781 785 self.inputId = '0'
782 786 self.parentId = parentId
783 787 self.server = server
784 788 self.addRunOperation(**kwargs)
785 789
786 790 def update(self, **kwargs):
787 791
788 792 if 'datatype' in kwargs:
789 793 datatype = kwargs.pop('datatype')
790 794 if 'Reader' in datatype:
791 795 self.name = datatype
792 796 else:
793 797 self.name = '%sReader' % (datatype)
794 798 self.datatype = self.name.replace('Reader', '')
795 799
796 800 attrs = ('path', 'startDate', 'endDate',
797 801 'startTime', 'endTime', 'parentId')
798 802
799 803 for attr in attrs:
800 804 if attr in kwargs:
801 805 setattr(self, attr, kwargs.pop(attr))
802 806
803 807 self.inputId = '0'
804 808 self.updateRunOperation(**kwargs)
805 809
806 810 def removeOperations(self):
807 811
808 812 for obj in self.opConfObjList:
809 813 del obj
810 814
811 815 self.opConfObjList = []
812 816
813 817 def addRunOperation(self, **kwargs):
814 818
815 819 opObj = self.addOperation(name='run', optype='self')
816 820
817 821 if self.server is None:
818 822 opObj.addParameter(
819 823 name='datatype', value=self.datatype, format='str')
820 824 opObj.addParameter(name='path', value=self.path, format='str')
821 825 opObj.addParameter(
822 826 name='startDate', value=self.startDate, format='date')
823 827 opObj.addParameter(
824 828 name='endDate', value=self.endDate, format='date')
825 829 opObj.addParameter(
826 830 name='startTime', value=self.startTime, format='time')
827 831 opObj.addParameter(
828 832 name='endTime', value=self.endTime, format='time')
829 833
830 834 for key, value in kwargs.items():
831 835 opObj.addParameter(name=key, value=value,
832 836 format=type(value).__name__)
833 837 else:
834 838 opObj.addParameter(name='server', value=self.server, format='str')
835 839
836 840 return opObj
837 841
838 842 def updateRunOperation(self, **kwargs):
839 843
840 844 opObj = self.getOperationObj(name='run')
841 845 opObj.removeParameters()
842 846
843 847 opObj.addParameter(name='datatype', value=self.datatype, format='str')
844 848 opObj.addParameter(name='path', value=self.path, format='str')
845 849 opObj.addParameter(
846 850 name='startDate', value=self.startDate, format='date')
847 851 opObj.addParameter(name='endDate', value=self.endDate, format='date')
848 852 opObj.addParameter(
849 853 name='startTime', value=self.startTime, format='time')
850 854 opObj.addParameter(name='endTime', value=self.endTime, format='time')
851 855
852 856 for key, value in kwargs.items():
853 857 opObj.addParameter(name=key, value=value,
854 858 format=type(value).__name__)
855 859
856 860 return opObj
857 861
858 862 def readXml(self, upElement):
859 863
860 864 self.id = upElement.get('id')
861 865 self.name = upElement.get('name')
862 866 self.datatype = upElement.get('datatype')
863 867 self.inputId = upElement.get('inputId')
864 868
865 869 if self.ELEMENTNAME == 'ReadUnit':
866 870 self.datatype = self.datatype.replace('Reader', '')
867 871
868 872 if self.inputId == 'None':
869 873 self.inputId = '0'
870 874
871 875 self.opConfObjList = []
872 876
873 877 opElementList = upElement.iter(OperationConf().getElementName())
874 878
875 879 for opElement in opElementList:
876 880 opConfObj = OperationConf()
877 881 opConfObj.readXml(opElement)
878 882 self.opConfObjList.append(opConfObj)
879 883
880 884 if opConfObj.name == 'run':
881 885 self.path = opConfObj.getParameterValue('path')
882 886 self.startDate = opConfObj.getParameterValue('startDate')
883 887 self.endDate = opConfObj.getParameterValue('endDate')
884 888 self.startTime = opConfObj.getParameterValue('startTime')
885 889 self.endTime = opConfObj.getParameterValue('endTime')
886 890
887 891
888 892 class Project(Process):
889 893
890 894 id = None
891 895 # name = None
892 896 description = None
893 897 filename = None
894 898
895 899 procUnitConfObjDict = None
896 900
897 901 ELEMENTNAME = 'Project'
898 902
899 903 plotterQueue = None
900 904
901 905 def __init__(self, plotter_queue=None):
902 906
903 907 Process.__init__(self)
904 908 self.id = None
905 909 # self.name = None
906 910 self.description = None
907 911
908 912 self.plotterQueue = plotter_queue
909 913
910 914 self.procUnitConfObjDict = {}
911 915
912 916 def __getNewId(self):
913 917
914 918 idList = self.procUnitConfObjDict.keys()
915 919
916 920 id = int(self.id) * 10
917 921
918 922 while True:
919 923 id += 1
920 924
921 925 if str(id) in idList:
922 926 continue
923 927
924 928 break
925 929
926 930 return str(id)
927 931
928 932 def getElementName(self):
929 933
930 934 return self.ELEMENTNAME
931 935
932 936 def getId(self):
933 937
934 938 return self.id
935 939
936 940 def updateId(self, new_id):
937 941
938 942 self.id = str(new_id)
939 943
940 944 keyList = self.procUnitConfObjDict.keys()
941 945 keyList.sort()
942 946
943 947 n = 1
944 948 newProcUnitConfObjDict = {}
945 949
946 950 for procKey in keyList:
947 951
948 952 procUnitConfObj = self.procUnitConfObjDict[procKey]
949 953 idProcUnit = str(int(self.id) * 10 + n)
950 954 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
951 955
952 956 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
953 957 n += 1
954 958
955 959 self.procUnitConfObjDict = newProcUnitConfObjDict
956 960
957 961 def setup(self, id, name='', description=''):
958 962
959 963 print
960 964 print '*' * 60
961 965 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
962 966 print '*' * 60
963 967 print
964 968 self.id = str(id)
965 self.description = description
969 self.description = description
966 970
967 971 def update(self, name, description):
968 972
969 973 self.description = description
970 974
971 975 def clone(self):
972 976
973 977 p = Project()
974 978 p.procUnitConfObjDict = self.procUnitConfObjDict
975 979 return p
976 980
977 981 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
978 982
979 983 if id is None:
980 984 idReadUnit = self.__getNewId()
981 985 else:
982 986 idReadUnit = str(id)
983 987
984 988 readUnitConfObj = ReadUnitConf()
985 989 readUnitConfObj.setup(idReadUnit, name, datatype,
986 990 parentId=self.id, **kwargs)
987 991
988 992 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
989 993
990 994 return readUnitConfObj
991 995
992 996 def addProcUnit(self, inputId='0', datatype=None, name=None):
993 997
994 998 idProcUnit = self.__getNewId()
995 999
996 1000 procUnitConfObj = ProcUnitConf()
997 1001 procUnitConfObj.setup(idProcUnit, name, datatype,
998 1002 inputId, parentId=self.id)
999 1003
1000 1004 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1001 1005
1002 1006 return procUnitConfObj
1003 1007
1004 1008 def removeProcUnit(self, id):
1005 1009
1006 1010 if id in self.procUnitConfObjDict.keys():
1007 1011 self.procUnitConfObjDict.pop(id)
1008 1012
1009 1013 def getReadUnitId(self):
1010 1014
1011 1015 readUnitConfObj = self.getReadUnitObj()
1012 1016
1013 1017 return readUnitConfObj.id
1014 1018
1015 1019 def getReadUnitObj(self):
1016 1020
1017 1021 for obj in self.procUnitConfObjDict.values():
1018 1022 if obj.getElementName() == 'ReadUnit':
1019 1023 return obj
1020 1024
1021 1025 return None
1022 1026
1023 1027 def getProcUnitObj(self, id=None, name=None):
1024 1028
1025 1029 if id != None:
1026 1030 return self.procUnitConfObjDict[id]
1027 1031
1028 1032 if name != None:
1029 1033 return self.getProcUnitObjByName(name)
1030 1034
1031 1035 return None
1032 1036
1033 1037 def getProcUnitObjByName(self, name):
1034 1038
1035 1039 for obj in self.procUnitConfObjDict.values():
1036 1040 if obj.name == name:
1037 1041 return obj
1038 1042
1039 1043 return None
1040 1044
1041 1045 def procUnitItems(self):
1042 1046
1043 1047 return self.procUnitConfObjDict.items()
1044 1048
1045 1049 def makeXml(self):
1046 1050
1047 1051 projectElement = Element('Project')
1048 1052 projectElement.set('id', str(self.id))
1049 1053 projectElement.set('name', self.name)
1050 1054 projectElement.set('description', self.description)
1051 1055
1052 1056 for procUnitConfObj in self.procUnitConfObjDict.values():
1053 1057 procUnitConfObj.makeXml(projectElement)
1054 1058
1055 1059 self.projectElement = projectElement
1056 1060
1057 1061 def writeXml(self, filename=None):
1058 1062
1059 1063 if filename == None:
1060 1064 if self.filename:
1061 1065 filename = self.filename
1062 1066 else:
1063 1067 filename = 'schain.xml'
1064 1068
1065 1069 if not filename:
1066 1070 print 'filename has not been defined. Use setFilename(filename) for do it.'
1067 1071 return 0
1068 1072
1069 1073 abs_file = os.path.abspath(filename)
1070 1074
1071 1075 if not os.access(os.path.dirname(abs_file), os.W_OK):
1072 1076 print 'No write permission on %s' % os.path.dirname(abs_file)
1073 1077 return 0
1074 1078
1075 1079 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1076 1080 print 'File %s already exists and it could not be overwriten' % abs_file
1077 1081 return 0
1078 1082
1079 1083 self.makeXml()
1080 1084
1081 1085 ElementTree(self.projectElement).write(abs_file, method='xml')
1082 1086
1083 1087 self.filename = abs_file
1084 1088
1085 1089 return 1
1086 1090
1087 1091 def readXml(self, filename=None):
1088 1092
1089 1093 if not filename:
1090 1094 print 'filename is not defined'
1091 1095 return 0
1092 1096
1093 1097 abs_file = os.path.abspath(filename)
1094 1098
1095 1099 if not os.path.isfile(abs_file):
1096 1100 print '%s file does not exist' % abs_file
1097 1101 return 0
1098 1102
1099 1103 self.projectElement = None
1100 1104 self.procUnitConfObjDict = {}
1101 1105
1102 1106 try:
1103 1107 self.projectElement = ElementTree().parse(abs_file)
1104 1108 except:
1105 1109 print 'Error reading %s, verify file format' % filename
1106 1110 return 0
1107 1111
1108 1112 self.project = self.projectElement.tag
1109 1113
1110 1114 self.id = self.projectElement.get('id')
1111 1115 self.name = self.projectElement.get('name')
1112 1116 self.description = self.projectElement.get('description')
1113 1117
1114 1118 readUnitElementList = self.projectElement.iter(
1115 1119 ReadUnitConf().getElementName())
1116 1120
1117 1121 for readUnitElement in readUnitElementList:
1118 1122 readUnitConfObj = ReadUnitConf()
1119 1123 readUnitConfObj.readXml(readUnitElement)
1120 1124
1121 1125 if readUnitConfObj.parentId == None:
1122 1126 readUnitConfObj.parentId = self.id
1123 1127
1124 1128 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1125 1129
1126 1130 procUnitElementList = self.projectElement.iter(
1127 1131 ProcUnitConf().getElementName())
1128 1132
1129 1133 for procUnitElement in procUnitElementList:
1130 1134 procUnitConfObj = ProcUnitConf()
1131 1135 procUnitConfObj.readXml(procUnitElement)
1132 1136
1133 1137 if procUnitConfObj.parentId == None:
1134 1138 procUnitConfObj.parentId = self.id
1135 1139
1136 1140 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1137 1141
1138 1142 self.filename = abs_file
1139 1143
1140 1144 return 1
1141 1145
1142 1146 def printattr(self):
1143 1147
1144 1148 print 'Project[%s]: name = %s, description = %s' % (self.id,
1145 1149 self.name,
1146 1150 self.description)
1147 1151
1148 1152 for procUnitConfObj in self.procUnitConfObjDict.values():
1149 1153 procUnitConfObj.printattr()
1150 1154
1151 1155 def createObjects(self):
1152 1156
1153 1157 for procUnitConfObj in self.procUnitConfObjDict.values():
1154 1158 procUnitConfObj.createObjects(self.plotterQueue)
1155 1159
1156 1160 def __connect(self, objIN, thisObj):
1157 1161
1158 1162 thisObj.setInput(objIN.getOutputObj())
1159 1163
1160 1164 def connectObjects(self):
1161 1165
1162 1166 for thisPUConfObj in self.procUnitConfObjDict.values():
1163 1167
1164 1168 inputId = thisPUConfObj.getInputId()
1165 1169
1166 1170 if int(inputId) == 0:
1167 1171 continue
1168 1172
1169 1173 # Get input object
1170 1174 puConfINObj = self.procUnitConfObjDict[inputId]
1171 1175 puObjIN = puConfINObj.getProcUnitObj()
1172 1176
1173 1177 # Get current object
1174 1178 thisPUObj = thisPUConfObj.getProcUnitObj()
1175 1179
1176 1180 self.__connect(puObjIN, thisPUObj)
1177 1181
1178 1182 def __handleError(self, procUnitConfObj, send_email=False):
1179 1183
1180 1184 import socket
1181 1185
1182 1186 err = traceback.format_exception(sys.exc_info()[0],
1183 1187 sys.exc_info()[1],
1184 1188 sys.exc_info()[2])
1185 1189
1186 1190 print '***** Error occurred in %s *****' % (procUnitConfObj.name)
1187 1191 print '***** %s' % err[-1]
1188 1192
1189 1193 message = ''.join(err)
1190 1194
1191 1195 sys.stderr.write(message)
1192 1196
1193 1197 if not send_email:
1194 1198 return
1195 1199
1196 1200 subject = 'SChain v%s: Error running %s\n' % (
1197 1201 schainpy.__version__, procUnitConfObj.name)
1198 1202
1199 1203 subtitle = '%s: %s\n' % (
1200 1204 procUnitConfObj.getElementName(), procUnitConfObj.name)
1201 1205 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1202 1206 socket.gethostname())
1203 1207 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1204 1208 subtitle += 'Configuration file: %s\n' % self.filename
1205 1209 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1206 1210
1207 1211 readUnitConfObj = self.getReadUnitObj()
1208 1212 if readUnitConfObj:
1209 1213 subtitle += '\nInput parameters:\n'
1210 1214 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1211 1215 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1212 1216 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1213 1217 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1214 1218 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1215 1219 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1216 1220
1217 1221 adminObj = schainpy.admin.SchainNotify()
1218 1222 adminObj.sendAlert(message=message,
1219 1223 subject=subject,
1220 1224 subtitle=subtitle,
1221 1225 filename=self.filename)
1222 1226
1223 1227 def isPaused(self):
1224 1228 return 0
1225 1229
1226 1230 def isStopped(self):
1227 1231 return 0
1228 1232
1229 1233 def runController(self):
1230 1234 '''
1231 1235 returns 0 when this process has been stopped, 1 otherwise
1232 1236 '''
1233 1237
1234 1238 if self.isPaused():
1235 1239 print 'Process suspended'
1236 1240
1237 1241 while True:
1238 1242 time.sleep(0.1)
1239 1243
1240 1244 if not self.isPaused():
1241 1245 break
1242 1246
1243 1247 if self.isStopped():
1244 1248 break
1245 1249
1246 1250 print 'Process reinitialized'
1247 1251
1248 1252 if self.isStopped():
1249 1253 print 'Process stopped'
1250 1254 return 0
1251 1255
1252 1256 return 1
1253 1257
1254 1258 def setFilename(self, filename):
1255 1259
1256 1260 self.filename = filename
1257 1261
1258 1262 def setPlotterQueue(self, plotter_queue):
1259 1263
1260 1264 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1261 1265
1262 1266 def getPlotterQueue(self):
1263 1267
1264 1268 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1265 1269
1266 1270 def useExternalPlotter(self):
1267 1271
1268 1272 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1269 1273
1270 1274 def run(self):
1271 1275
1272 1276 log.success('Starting {}'.format(self.name))
1273
1277 self.start_time = time.time()
1274 1278 self.createObjects()
1275 1279 self.connectObjects()
1276 1280
1277 1281 keyList = self.procUnitConfObjDict.keys()
1278 1282 keyList.sort()
1279 1283
1280 1284 while(True):
1281 1285
1282 1286 is_ok = False
1283 1287
1284 1288 for procKey in keyList:
1285 1289
1286 1290 procUnitConfObj = self.procUnitConfObjDict[procKey]
1287 1291
1288 1292 try:
1289 1293 sts = procUnitConfObj.run()
1290 1294 is_ok = is_ok or sts
1291 1295 except KeyboardInterrupt:
1292 1296 is_ok = False
1293 1297 break
1294 1298 except ValueError, e:
1295 1299 time.sleep(0.5)
1296 1300 self.__handleError(procUnitConfObj, send_email=True)
1297 1301 is_ok = False
1298 1302 break
1299 1303 except:
1300 1304 time.sleep(0.5)
1301 1305 self.__handleError(procUnitConfObj)
1302 1306 is_ok = False
1303 1307 break
1304 1308
1305 1309 # If every process unit finished so end process
1306 1310 if not(is_ok):
1307 1311 break
1308 1312
1309 1313 if not self.runController():
1310 1314 break
1311 1315
1312 1316 # Closing every process
1313 1317 for procKey in keyList:
1314 1318 procUnitConfObj = self.procUnitConfObjDict[procKey]
1315 1319 procUnitConfObj.close()
1316 1320
1317 log.success('{} finished'.format(self.name))
1321 log.success('{} finished (time: {}s)'.format(
1322 self.name,
1323 time.time()-self.start_time))
Show file before | Show file after | Hide comments
@@ -1,40 +1,39
1 1 #!/usr/bin/env python
2 2 import os
3 3 import sys
4 4 from schainpy.utils import log
5 5
6 6 try:
7 7 from PyQt4 import QtCore, QtGui
8 8 from PyQt4.QtGui import QApplication
9 9 except:
10 10 log.error(
11 'You should install PtQt4 module in order to run the GUI. See the README.')
11 'You should install PyQt4 module in order to run the GUI. See the README.')
12 12 sys.exit()
13 13
14
15 14 from schainpy.gui.viewcontroller.initwindow import InitWindow
16 15 from schainpy.gui.viewcontroller.basicwindow import BasicWindow
17 16 from schainpy.gui.viewcontroller.workspace import Workspace
18 17
19 18
20 19 def main():
21 20
22 21 app = QtGui.QApplication(sys.argv)
23 22
24 23 Welcome = InitWindow()
25 24
26 25 if not Welcome.exec_():
27 26 sys.exit(-1)
28 27
29 28 WorkPathspace = Workspace()
30 29 if not WorkPathspace.exec_():
31 30 sys.exit(-1)
32 31
33 32 MainGUI = BasicWindow()
34 33 MainGUI.setWorkSpaceGUI(WorkPathspace.dirComBox.currentText())
35 34 MainGUI.show()
36 35 sys.exit(app.exec_())
37 36
38 37
39 38 if __name__ == "__main__":
40 39 main()
Show file before | Show file after | Hide comments
@@ -1,905 +1,905
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
5 5 '''
6 6 import sys
7 7 import numpy
8 8 import copy
9 9 import datetime
10 10 import inspect
11 11
12 12 SPEED_OF_LIGHT = 299792458
13 13 SPEED_OF_LIGHT = 3e8
14 14
15 15 BASIC_STRUCTURE = numpy.dtype([
16 16 ('nSize', '<u4'),
17 17 ('nVersion', '<u2'),
18 18 ('nDataBlockId', '<u4'),
19 19 ('nUtime', '<u4'),
20 20 ('nMilsec', '<u2'),
21 21 ('nTimezone', '<i2'),
22 22 ('nDstflag', '<i2'),
23 23 ('nErrorCount', '<u4')
24 24 ])
25 25
26 26 SYSTEM_STRUCTURE = numpy.dtype([
27 27 ('nSize', '<u4'),
28 28 ('nNumSamples', '<u4'),
29 29 ('nNumProfiles', '<u4'),
30 30 ('nNumChannels', '<u4'),
31 31 ('nADCResolution', '<u4'),
32 32 ('nPCDIOBusWidth', '<u4'),
33 33 ])
34 34
35 35 RADAR_STRUCTURE = numpy.dtype([
36 36 ('nSize', '<u4'),
37 37 ('nExpType', '<u4'),
38 38 ('nNTx', '<u4'),
39 39 ('fIpp', '<f4'),
40 40 ('fTxA', '<f4'),
41 41 ('fTxB', '<f4'),
42 42 ('nNumWindows', '<u4'),
43 43 ('nNumTaus', '<u4'),
44 44 ('nCodeType', '<u4'),
45 45 ('nLine6Function', '<u4'),
46 46 ('nLine5Function', '<u4'),
47 47 ('fClock', '<f4'),
48 48 ('nPrePulseBefore', '<u4'),
49 49 ('nPrePulseAfter', '<u4'),
50 50 ('sRangeIPP', '<a20'),
51 51 ('sRangeTxA', '<a20'),
52 52 ('sRangeTxB', '<a20'),
53 53 ])
54 54
55 55 SAMPLING_STRUCTURE = numpy.dtype(
56 56 [('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')])
57 57
58 58
59 59 PROCESSING_STRUCTURE = numpy.dtype([
60 60 ('nSize', '<u4'),
61 61 ('nDataType', '<u4'),
62 62 ('nSizeOfDataBlock', '<u4'),
63 63 ('nProfilesperBlock', '<u4'),
64 64 ('nDataBlocksperFile', '<u4'),
65 65 ('nNumWindows', '<u4'),
66 66 ('nProcessFlags', '<u4'),
67 67 ('nCoherentIntegrations', '<u4'),
68 68 ('nIncoherentIntegrations', '<u4'),
69 69 ('nTotalSpectra', '<u4')
70 70 ])
71 71
72 72
73 73 class Header(object):
74 74
75 75 def __init__(self):
76 76 raise NotImplementedError
77 77
78 78 def copy(self):
79 79 return copy.deepcopy(self)
80 80
81 81 def read(self):
82 82
83 83 raise NotImplementedError
84 84
85 85 def write(self):
86 86
87 87 raise NotImplementedError
88 88
89 89 def getAllowedArgs(self):
90 90 args = inspect.getargspec(self.__init__).args
91 91 try:
92 92 args.remove('self')
93 93 except:
94 94 pass
95 95 return args
96 96
97 97 def getAsDict(self):
98 98 args = self.getAllowedArgs()
99 99 asDict = {}
100 100 for x in args:
101 101 asDict[x] = self[x]
102 102 return asDict
103 103
104 104 def __getitem__(self, name):
105 105 return getattr(self, name)
106 106
107 107 def printInfo(self):
108 108
109 109 message = "#" * 50 + "\n"
110 110 message += self.__class__.__name__.upper() + "\n"
111 111 message += "#" * 50 + "\n"
112 112
113 113 keyList = self.__dict__.keys()
114 114 keyList.sort()
115 115
116 116 for key in keyList:
117 117 message += "%s = %s" % (key, self.__dict__[key]) + "\n"
118 118
119 119 if "size" not in keyList:
120 120 attr = getattr(self, "size")
121 121
122 122 if attr:
123 123 message += "%s = %s" % ("size", attr) + "\n"
124 124
125 125 print message
126 126
127 127
128 128 class BasicHeader(Header):
129 129
130 130 size = None
131 131 version = None
132 132 dataBlock = None
133 133 utc = None
134 134 ltc = None
135 135 miliSecond = None
136 136 timeZone = None
137 137 dstFlag = None
138 138 errorCount = None
139 139 datatime = None
140 140 structure = BASIC_STRUCTURE
141 141 __LOCALTIME = None
142 142
143 143 def __init__(self, useLocalTime=True):
144 144
145 145 self.size = 24
146 146 self.version = 0
147 147 self.dataBlock = 0
148 148 self.utc = 0
149 149 self.miliSecond = 0
150 150 self.timeZone = 0
151 151 self.dstFlag = 0
152 152 self.errorCount = 0
153 153
154 154 self.useLocalTime = useLocalTime
155 155
156 156 def read(self, fp):
157 157
158 158 self.length = 0
159 159 try:
160 160 if hasattr(fp, 'read'):
161 161 header = numpy.fromfile(fp, BASIC_STRUCTURE, 1)
162 162 else:
163 163 header = numpy.fromstring(fp, BASIC_STRUCTURE, 1)
164 164 except Exception, e:
165 165 print "BasicHeader: "
166 166 print e
167 167 return 0
168 168
169 169 self.size = int(header['nSize'][0])
170 170 self.version = int(header['nVersion'][0])
171 171 self.dataBlock = int(header['nDataBlockId'][0])
172 172 self.utc = int(header['nUtime'][0])
173 173 self.miliSecond = int(header['nMilsec'][0])
174 174 self.timeZone = int(header['nTimezone'][0])
175 175 self.dstFlag = int(header['nDstflag'][0])
176 176 self.errorCount = int(header['nErrorCount'][0])
177 177
178 178 if self.size < 24:
179 179 return 0
180 180
181 181 self.length = header.nbytes
182 182 return 1
183 183
184 184 def write(self, fp):
185 185
186 186 headerTuple = (self.size, self.version, self.dataBlock, self.utc,
187 187 self.miliSecond, self.timeZone, self.dstFlag, self.errorCount)
188 188 header = numpy.array(headerTuple, BASIC_STRUCTURE)
189 189 header.tofile(fp)
190 190
191 191 return 1
192 192
193 193 def get_ltc(self):
194 194
195 195 return self.utc - self.timeZone * 60
196 196
197 197 def set_ltc(self, value):
198 198
199 199 self.utc = value + self.timeZone * 60
200 200
201 201 def get_datatime(self):
202 202
203 203 return datetime.datetime.utcfromtimestamp(self.ltc)
204 204
205 205 ltc = property(get_ltc, set_ltc)
206 206 datatime = property(get_datatime)
207 207
208 208
209 209 class SystemHeader(Header):
210 210
211 211 size = None
212 212 nSamples = None
213 213 nProfiles = None
214 214 nChannels = None
215 215 adcResolution = None
216 216 pciDioBusWidth = None
217 217 structure = SYSTEM_STRUCTURE
218 218
219 219 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWidth=0):
220 220
221 221 self.size = 24
222 222 self.nSamples = nSamples
223 223 self.nProfiles = nProfiles
224 224 self.nChannels = nChannels
225 225 self.adcResolution = adcResolution
226 226 self.pciDioBusWidth = pciDioBusWidth
227 227
228 228 def read(self, fp):
229 229 self.length = 0
230 230 try:
231 231 startFp = fp.tell()
232 232 except Exception, e:
233 233 startFp = None
234 234 pass
235 235
236 236 try:
237 237 if hasattr(fp, 'read'):
238 238 header = numpy.fromfile(fp, SYSTEM_STRUCTURE, 1)
239 239 else:
240 240 header = numpy.fromstring(fp, SYSTEM_STRUCTURE, 1)
241 241 except Exception, e:
242 242 print "System Header: " + str(e)
243 243 return 0
244 244
245 245 self.size = header['nSize'][0]
246 246 self.nSamples = header['nNumSamples'][0]
247 247 self.nProfiles = header['nNumProfiles'][0]
248 248 self.nChannels = header['nNumChannels'][0]
249 249 self.adcResolution = header['nADCResolution'][0]
250 250 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
251 251
252 252 if startFp is not None:
253 253 endFp = self.size + startFp
254 254
255 255 if fp.tell() > endFp:
256 256 sys.stderr.write(
257 257 "Warning %s: Size value read from System Header is lower than it has to be\n" % fp.name)
258 258 return 0
259 259
260 260 if fp.tell() < endFp:
261 261 sys.stderr.write(
262 262 "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp.name)
263 263 return 0
264 264
265 265 self.length = header.nbytes
266 266 return 1
267 267
268 268 def write(self, fp):
269 269
270 270 headerTuple = (self.size, self.nSamples, self.nProfiles,
271 271 self.nChannels, self.adcResolution, self.pciDioBusWidth)
272 272 header = numpy.array(headerTuple, SYSTEM_STRUCTURE)
273 273 header.tofile(fp)
274 274
275 275 return 1
276 276
277 277
278 278 class RadarControllerHeader(Header):
279 279
280 280 expType = None
281 281 nTx = None
282 282 ipp = None
283 283 txA = None
284 284 txB = None
285 285 nWindows = None
286 286 numTaus = None
287 287 codeType = None
288 288 line6Function = None
289 289 line5Function = None
290 290 fClock = None
291 291 prePulseBefore = None
292 292 prePulseAfter = None
293 293 rangeIpp = None
294 294 rangeTxA = None
295 295 rangeTxB = None
296 296 structure = RADAR_STRUCTURE
297 297 __size = None
298 298
299 299 def __init__(self, expType=2, nTx=1,
300 300 ipp=None, txA=0, txB=0,
301 301 nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
302 302 numTaus=0, line6Function=0, line5Function=0, fClock=None,
303 303 prePulseBefore=0, prePulseAfter=0,
304 304 codeType=0, nCode=0, nBaud=0, code=None,
305 305 flip1=0, flip2=0):
306 306
307 307 # self.size = 116
308 308 self.expType = expType
309 309 self.nTx = nTx
310 310 self.ipp = ipp
311 311 self.txA = txA
312 312 self.txB = txB
313 313 self.rangeIpp = ipp
314 314 self.rangeTxA = txA
315 315 self.rangeTxB = txB
316 316
317 317 self.nWindows = nWindows
318 318 self.numTaus = numTaus
319 319 self.codeType = codeType
320 320 self.line6Function = line6Function
321 321 self.line5Function = line5Function
322 322 self.fClock = fClock
323 323 self.prePulseBefore = prePulseBefore
324 324 self.prePulseAfter = prePulseAfter
325 325
326 326 self.nHeights = nHeights
327 327 self.firstHeight = firstHeight
328 328 self.deltaHeight = deltaHeight
329 329 self.samplesWin = nHeights
330 330
331 331 self.nCode = nCode
332 332 self.nBaud = nBaud
333 333 self.code = code
334 334 self.flip1 = flip1
335 335 self.flip2 = flip2
336 336
337 337 self.code_size = int(numpy.ceil(self.nBaud / 32.)) * self.nCode * 4
338 338 # self.dynamic = numpy.array([],numpy.dtype('byte'))
339 339
340 340 if self.fClock is None and self.deltaHeight is not None:
341 341 self.fClock = 0.15 / (deltaHeight * 1e-6) # 0.15Km / (height * 1u)
342 342
343 343 def read(self, fp):
344 344 self.length = 0
345 345 try:
346 346 startFp = fp.tell()
347 347 except Exception, e:
348 348 startFp = None
349 349 pass
350 350
351 351 try:
352 352 if hasattr(fp, 'read'):
353 353 header = numpy.fromfile(fp, RADAR_STRUCTURE, 1)
354 354 else:
355 355 header = numpy.fromstring(fp, RADAR_STRUCTURE, 1)
356 356 self.length += header.nbytes
357 357 except Exception, e:
358 358 print "RadarControllerHeader: " + str(e)
359 359 return 0
360 360
361 361 size = int(header['nSize'][0])
362 362 self.expType = int(header['nExpType'][0])
363 363 self.nTx = int(header['nNTx'][0])
364 364 self.ipp = float(header['fIpp'][0])
365 365 self.txA = float(header['fTxA'][0])
366 366 self.txB = float(header['fTxB'][0])
367 367 self.nWindows = int(header['nNumWindows'][0])
368 368 self.numTaus = int(header['nNumTaus'][0])
369 369 self.codeType = int(header['nCodeType'][0])
370 370 self.line6Function = int(header['nLine6Function'][0])
371 371 self.line5Function = int(header['nLine5Function'][0])
372 372 self.fClock = float(header['fClock'][0])
373 373 self.prePulseBefore = int(header['nPrePulseBefore'][0])
374 374 self.prePulseAfter = int(header['nPrePulseAfter'][0])
375 375 self.rangeIpp = header['sRangeIPP'][0]
376 376 self.rangeTxA = header['sRangeTxA'][0]
377 377 self.rangeTxB = header['sRangeTxB'][0]
378 378
379 379 try:
380 380 if hasattr(fp, 'read'):
381 381 samplingWindow = numpy.fromfile(
382 382 fp, SAMPLING_STRUCTURE, self.nWindows)
383 383 else:
384 384 samplingWindow = numpy.fromstring(
385 385 fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
386 386 self.length += samplingWindow.nbytes
387 387 except Exception, e:
388 388 print "RadarControllerHeader: " + str(e)
389 389 return 0
390 390 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
391 391 self.firstHeight = samplingWindow['h0']
392 392 self.deltaHeight = samplingWindow['dh']
393 393 self.samplesWin = samplingWindow['nsa']
394 394
395 395 try:
396 396 if hasattr(fp, 'read'):
397 397 self.Taus = numpy.fromfile(fp, '<f4', self.numTaus)
398 398 else:
399 399 self.Taus = numpy.fromstring(
400 400 fp[self.length:], '<f4', self.numTaus)
401 401 self.length += self.Taus.nbytes
402 402 except Exception, e:
403 403 print "RadarControllerHeader: " + str(e)
404 404 return 0
405 405
406 406 self.code_size = 0
407 407 if self.codeType != 0:
408 408
409 409 try:
410 410 if hasattr(fp, 'read'):
411 411 self.nCode = numpy.fromfile(fp, '<u4', 1)[0]
412 412 self.length += self.nCode.nbytes
413 413 self.nBaud = numpy.fromfile(fp, '<u4', 1)[0]
414 414 self.length += self.nBaud.nbytes
415 415 else:
416 416 self.nCode = numpy.fromstring(
417 417 fp[self.length:], '<u4', 1)[0]
418 418 self.length += self.nCode.nbytes
419 419 self.nBaud = numpy.fromstring(
420 420 fp[self.length:], '<u4', 1)[0]
421 421 self.length += self.nBaud.nbytes
422 422 except Exception, e:
423 423 print "RadarControllerHeader: " + str(e)
424 424 return 0
425 425 code = numpy.empty([self.nCode, self.nBaud], dtype='i1')
426 426
427 427 for ic in range(self.nCode):
428 428 try:
429 429 if hasattr(fp, 'read'):
430 430 temp = numpy.fromfile(fp, 'u4', int(
431 431 numpy.ceil(self.nBaud / 32.)))
432 432 else:
433 433 temp = numpy.fromstring(
434 434 fp, 'u4', int(numpy.ceil(self.nBaud / 32.)))
435 435 self.length += temp.nbytes
436 436 except Exception, e:
437 437 print "RadarControllerHeader: " + str(e)
438 438 return 0
439 439
440 440 for ib in range(self.nBaud - 1, -1, -1):
441 441 code[ic, ib] = temp[ib / 32] % 2
442 442 temp[ib / 32] = temp[ib / 32] / 2
443 443
444 444 self.code = 2.0 * code - 1.0
445 445 self.code_size = int(numpy.ceil(self.nBaud / 32.)) * self.nCode * 4
446 446
447 447 # if self.line5Function == RCfunction.FLIP:
448 448 # self.flip1 = numpy.fromfile(fp,'<u4',1)
449 449 #
450 450 # if self.line6Function == RCfunction.FLIP:
451 451 # self.flip2 = numpy.fromfile(fp,'<u4',1)
452 452 if startFp is not None:
453 453 endFp = size + startFp
454 454
455 455 if fp.tell() != endFp:
456 456 # fp.seek(endFp)
457 457 print "%s: Radar Controller Header size is not consistent: from data [%d] != from header field [%d]" % (fp.name, fp.tell() - startFp, size)
458 458 # return 0
459 459
460 460 if fp.tell() > endFp:
461 461 sys.stderr.write(
462 462 "Warning %s: Size value read from Radar Controller header is lower than it has to be\n" % fp.name)
463 463 # return 0
464 464
465 465 if fp.tell() < endFp:
466 466 sys.stderr.write(
467 467 "Warning %s: Size value read from Radar Controller header is greater than it has to be\n" % fp.name)
468 468
469 469 return 1
470 470
471 471 def write(self, fp):
472 472
473 473 headerTuple = (self.size,
474 474 self.expType,
475 475 self.nTx,
476 476 self.ipp,
477 477 self.txA,
478 478 self.txB,
479 479 self.nWindows,
480 480 self.numTaus,
481 481 self.codeType,
482 482 self.line6Function,
483 483 self.line5Function,
484 484 self.fClock,
485 485 self.prePulseBefore,
486 486 self.prePulseAfter,
487 487 self.rangeIpp,
488 488 self.rangeTxA,
489 489 self.rangeTxB)
490 490
491 491 header = numpy.array(headerTuple, RADAR_STRUCTURE)
492 492 header.tofile(fp)
493 493
494 494 sampleWindowTuple = (
495 495 self.firstHeight, self.deltaHeight, self.samplesWin)
496 496 samplingWindow = numpy.array(sampleWindowTuple, SAMPLING_STRUCTURE)
497 497 samplingWindow.tofile(fp)
498 498
499 499 if self.numTaus > 0:
500 500 self.Taus.tofile(fp)
501 501
502 502 if self.codeType != 0:
503 503 nCode = numpy.array(self.nCode, '<u4')
504 504 nCode.tofile(fp)
505 505 nBaud = numpy.array(self.nBaud, '<u4')
506 506 nBaud.tofile(fp)
507 507 code1 = (self.code + 1.0) / 2.
508 508
509 509 for ic in range(self.nCode):
510 tempx = numpy.zeros(int(self.nBaud / 32.))
510 tempx = numpy.zeros(int(numpy.ceil(self.nBaud / 32.)))
511 511 start = 0
512 512 end = 32
513 513 for i in range(len(tempx)):
514 514 code_selected = code1[ic, start:end]
515 515 for j in range(len(code_selected) - 1, -1, -1):
516 516 if code_selected[j] == 1:
517 517 tempx[i] = tempx[i] + \
518 518 2**(len(code_selected) - 1 - j)
519 519 start = start + 32
520 520 end = end + 32
521 521
522 522 tempx = tempx.astype('u4')
523 523 tempx.tofile(fp)
524 524
525 525 # if self.line5Function == RCfunction.FLIP:
526 526 # self.flip1.tofile(fp)
527 527 #
528 528 # if self.line6Function == RCfunction.FLIP:
529 529 # self.flip2.tofile(fp)
530 530
531 531 return 1
532 532
533 533 def get_ippSeconds(self):
534 534 '''
535 535 '''
536 536 ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT
537 537
538 538 return ippSeconds
539 539
540 540 def set_ippSeconds(self, ippSeconds):
541 541 '''
542 542 '''
543 543
544 544 self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0 * 1000)
545 545
546 546 return
547 547
548 548 def get_size(self):
549 549
550 550 self.__size = 116 + 12 * self.nWindows + 4 * self.numTaus
551 551
552 552 if self.codeType != 0:
553 553 self.__size += 4 + 4 + 4 * self.nCode * \
554 554 numpy.ceil(self.nBaud / 32.)
555 555
556 556 return self.__size
557 557
558 558 def set_size(self, value):
559 559
560 560 raise IOError, "size is a property and it cannot be set, just read"
561 561
562 562 return
563 563
564 564 ippSeconds = property(get_ippSeconds, set_ippSeconds)
565 565 size = property(get_size, set_size)
566 566
567 567
568 568 class ProcessingHeader(Header):
569 569
570 570 # size = None
571 571 dtype = None
572 572 blockSize = None
573 573 profilesPerBlock = None
574 574 dataBlocksPerFile = None
575 575 nWindows = None
576 576 processFlags = None
577 577 nCohInt = None
578 578 nIncohInt = None
579 579 totalSpectra = None
580 580 structure = PROCESSING_STRUCTURE
581 581 flag_dc = None
582 582 flag_cspc = None
583 583
584 584 def __init__(self, dtype=0, blockSize=0, profilesPerBlock=0, dataBlocksPerFile=0, nWindows=0, processFlags=0, nCohInt=0,
585 585 nIncohInt=0, totalSpectra=0, nHeights=0, firstHeight=0, deltaHeight=0, samplesWin=0, spectraComb=0, nCode=0,
586 586 code=0, nBaud=None, shif_fft=False, flag_dc=False, flag_cspc=False, flag_decode=False, flag_deflip=False
587 587 ):
588 588
589 589 # self.size = 0
590 590 self.dtype = dtype
591 591 self.blockSize = blockSize
592 592 self.profilesPerBlock = 0
593 593 self.dataBlocksPerFile = 0
594 594 self.nWindows = 0
595 595 self.processFlags = 0
596 596 self.nCohInt = 0
597 597 self.nIncohInt = 0
598 598 self.totalSpectra = 0
599 599
600 600 self.nHeights = 0
601 601 self.firstHeight = 0
602 602 self.deltaHeight = 0
603 603 self.samplesWin = 0
604 604 self.spectraComb = 0
605 605 self.nCode = None
606 606 self.code = None
607 607 self.nBaud = None
608 608
609 609 self.shif_fft = False
610 610 self.flag_dc = False
611 611 self.flag_cspc = False
612 612 self.flag_decode = False
613 613 self.flag_deflip = False
614 614 self.length = 0
615 615
616 616 def read(self, fp):
617 617 self.length = 0
618 618 try:
619 619 startFp = fp.tell()
620 620 except Exception, e:
621 621 startFp = None
622 622 pass
623 623
624 624 try:
625 625 if hasattr(fp, 'read'):
626 626 header = numpy.fromfile(fp, PROCESSING_STRUCTURE, 1)
627 627 else:
628 628 header = numpy.fromstring(fp, PROCESSING_STRUCTURE, 1)
629 629 self.length += header.nbytes
630 630 except Exception, e:
631 631 print "ProcessingHeader: " + str(e)
632 632 return 0
633 633
634 634 size = int(header['nSize'][0])
635 635 self.dtype = int(header['nDataType'][0])
636 636 self.blockSize = int(header['nSizeOfDataBlock'][0])
637 637 self.profilesPerBlock = int(header['nProfilesperBlock'][0])
638 638 self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0])
639 639 self.nWindows = int(header['nNumWindows'][0])
640 640 self.processFlags = header['nProcessFlags']
641 641 self.nCohInt = int(header['nCoherentIntegrations'][0])
642 642 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
643 643 self.totalSpectra = int(header['nTotalSpectra'][0])
644 644
645 645 try:
646 646 if hasattr(fp, 'read'):
647 647 samplingWindow = numpy.fromfile(
648 648 fp, SAMPLING_STRUCTURE, self.nWindows)
649 649 else:
650 650 samplingWindow = numpy.fromstring(
651 651 fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
652 652 self.length += samplingWindow.nbytes
653 653 except Exception, e:
654 654 print "ProcessingHeader: " + str(e)
655 655 return 0
656 656
657 657 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
658 658 self.firstHeight = float(samplingWindow['h0'][0])
659 659 self.deltaHeight = float(samplingWindow['dh'][0])
660 660 self.samplesWin = samplingWindow['nsa'][0]
661 661
662 662 try:
663 663 if hasattr(fp, 'read'):
664 664 self.spectraComb = numpy.fromfile(
665 665 fp, 'u1', 2 * self.totalSpectra)
666 666 else:
667 667 self.spectraComb = numpy.fromstring(
668 668 fp[self.length:], 'u1', 2 * self.totalSpectra)
669 669 self.length += self.spectraComb.nbytes
670 670 except Exception, e:
671 671 print "ProcessingHeader: " + str(e)
672 672 return 0
673 673
674 674 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
675 675 self.nCode = int(numpy.fromfile(fp, '<u4', 1))
676 676 self.nBaud = int(numpy.fromfile(fp, '<u4', 1))
677 677 self.code = numpy.fromfile(
678 678 fp, '<f4', self.nCode * self.nBaud).reshape(self.nCode, self.nBaud)
679 679
680 680 if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP):
681 681 exp_name_len = int(numpy.fromfile(fp, '<u4', 1))
682 682 exp_name = numpy.fromfile(fp, 'u1', exp_name_len + 1)
683 683
684 684 if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
685 685 self.shif_fft = True
686 686 else:
687 687 self.shif_fft = False
688 688
689 689 if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
690 690 self.flag_dc = True
691 691 else:
692 692 self.flag_dc = False
693 693
694 694 if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA):
695 695 self.flag_decode = True
696 696 else:
697 697 self.flag_decode = False
698 698
699 699 if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA):
700 700 self.flag_deflip = True
701 701 else:
702 702 self.flag_deflip = False
703 703
704 704 nChannels = 0
705 705 nPairs = 0
706 706 pairList = []
707 707
708 708 for i in range(0, self.totalSpectra * 2, 2):
709 709 if self.spectraComb[i] == self.spectraComb[i + 1]:
710 710 nChannels = nChannels + 1 # par de canales iguales
711 711 else:
712 712 nPairs = nPairs + 1 # par de canales diferentes
713 713 pairList.append((self.spectraComb[i], self.spectraComb[i + 1]))
714 714
715 715 self.flag_cspc = False
716 716 if nPairs > 0:
717 717 self.flag_cspc = True
718 718
719 719 if startFp is not None:
720 720 endFp = size + startFp
721 721 if fp.tell() > endFp:
722 722 sys.stderr.write(
723 723 "Warning: Processing header size is lower than it has to be")
724 724 return 0
725 725
726 726 if fp.tell() < endFp:
727 727 sys.stderr.write(
728 728 "Warning: Processing header size is greater than it is considered")
729 729
730 730 return 1
731 731
732 732 def write(self, fp):
733 733 # Clear DEFINE_PROCESS_CODE
734 734 self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE)
735 735
736 736 headerTuple = (self.size,
737 737 self.dtype,
738 738 self.blockSize,
739 739 self.profilesPerBlock,
740 740 self.dataBlocksPerFile,
741 741 self.nWindows,
742 742 self.processFlags,
743 743 self.nCohInt,
744 744 self.nIncohInt,
745 745 self.totalSpectra)
746 746
747 747 header = numpy.array(headerTuple, PROCESSING_STRUCTURE)
748 748 header.tofile(fp)
749 749
750 750 if self.nWindows != 0:
751 751 sampleWindowTuple = (
752 752 self.firstHeight, self.deltaHeight, self.samplesWin)
753 753 samplingWindow = numpy.array(sampleWindowTuple, SAMPLING_STRUCTURE)
754 754 samplingWindow.tofile(fp)
755 755
756 756 if self.totalSpectra != 0:
757 757 # spectraComb = numpy.array([],numpy.dtype('u1'))
758 758 spectraComb = self.spectraComb
759 759 spectraComb.tofile(fp)
760 760
761 761 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
762 762 # nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba
763 763 # nCode.tofile(fp)
764 764 #
765 765 # nBaud = numpy.array([self.nBaud], numpy.dtype('u4'))
766 766 # nBaud.tofile(fp)
767 767 #
768 768 # code = self.code.reshape(self.nCode*self.nBaud)
769 769 # code = code.astype(numpy.dtype('<f4'))
770 770 # code.tofile(fp)
771 771
772 772 return 1
773 773
774 774 def get_size(self):
775 775
776 776 self.__size = 40 + 12 * self.nWindows + 2 * self.totalSpectra
777 777
778 778 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
779 779 # self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
780 780 # self.__size += 4 + 4 + 4 * self.nCode * self.nBaud
781 781
782 782 return self.__size
783 783
784 784 def set_size(self, value):
785 785
786 786 raise IOError, "size is a property and it cannot be set, just read"
787 787
788 788 return
789 789
790 790 size = property(get_size, set_size)
791 791
792 792
793 793 class RCfunction:
794 794 NONE = 0
795 795 FLIP = 1
796 796 CODE = 2
797 797 SAMPLING = 3
798 798 LIN6DIV256 = 4
799 799 SYNCHRO = 5
800 800
801 801
802 802 class nCodeType:
803 803 NONE = 0
804 804 USERDEFINE = 1
805 805 BARKER2 = 2
806 806 BARKER3 = 3
807 807 BARKER4 = 4
808 808 BARKER5 = 5
809 809 BARKER7 = 6
810 810 BARKER11 = 7
811 811 BARKER13 = 8
812 812 AC128 = 9
813 813 COMPLEMENTARYCODE2 = 10
814 814 COMPLEMENTARYCODE4 = 11
815 815 COMPLEMENTARYCODE8 = 12
816 816 COMPLEMENTARYCODE16 = 13
817 817 COMPLEMENTARYCODE32 = 14
818 818 COMPLEMENTARYCODE64 = 15
819 819 COMPLEMENTARYCODE128 = 16
820 820 CODE_BINARY28 = 17
821 821
822 822
823 823 class PROCFLAG:
824 824
825 825 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
826 826 DECODE_DATA = numpy.uint32(0x00000002)
827 827 SPECTRA_CALC = numpy.uint32(0x00000004)
828 828 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
829 829 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
830 830 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
831 831
832 832 DATATYPE_CHAR = numpy.uint32(0x00000040)
833 833 DATATYPE_SHORT = numpy.uint32(0x00000080)
834 834 DATATYPE_LONG = numpy.uint32(0x00000100)
835 835 DATATYPE_INT64 = numpy.uint32(0x00000200)
836 836 DATATYPE_FLOAT = numpy.uint32(0x00000400)
837 837 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
838 838
839 839 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
840 840 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
841 841 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
842 842
843 843 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
844 844 DEFLIP_DATA = numpy.uint32(0x00010000)
845 845 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
846 846
847 847 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
848 848 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
849 849 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
850 850 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
851 851 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
852 852
853 853 EXP_NAME_ESP = numpy.uint32(0x00200000)
854 854 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
855 855
856 856 OPERATION_MASK = numpy.uint32(0x0000003F)
857 857 DATATYPE_MASK = numpy.uint32(0x00000FC0)
858 858 DATAARRANGE_MASK = numpy.uint32(0x00007000)
859 859 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
860 860
861 861
862 862 dtype0 = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
863 863 dtype1 = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
864 864 dtype2 = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
865 865 dtype3 = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
866 866 dtype4 = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
867 867 dtype5 = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
868 868
869 869 NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
870 870
871 871 PROCFLAG_DTYPE_LIST = [PROCFLAG.DATATYPE_CHAR,
872 872 PROCFLAG.DATATYPE_SHORT,
873 873 PROCFLAG.DATATYPE_LONG,
874 874 PROCFLAG.DATATYPE_INT64,
875 875 PROCFLAG.DATATYPE_FLOAT,
876 876 PROCFLAG.DATATYPE_DOUBLE]
877 877
878 878 DTYPE_WIDTH = [1, 2, 4, 8, 4, 8]
879 879
880 880
881 881 def get_dtype_index(numpy_dtype):
882 882
883 883 index = None
884 884
885 885 for i in range(len(NUMPY_DTYPE_LIST)):
886 886 if numpy_dtype == NUMPY_DTYPE_LIST[i]:
887 887 index = i
888 888 break
889 889
890 890 return index
891 891
892 892
893 893 def get_numpy_dtype(index):
894 894
895 895 return NUMPY_DTYPE_LIST[index]
896 896
897 897
898 898 def get_procflag_dtype(index):
899 899
900 900 return PROCFLAG_DTYPE_LIST[index]
901 901
902 902
903 903 def get_dtype_width(index):
904 904
905 905 return DTYPE_WIDTH[index]
Show file before | Show file after | Hide comments
@@ -1,955 +1,965
1 1
2 2 import os
3 3 import time
4 4 import glob
5 5 import datetime
6 6 from multiprocessing import Process
7 7
8 8 import zmq
9 9 import numpy
10 10 import matplotlib
11 11 import matplotlib.pyplot as plt
12 12 from mpl_toolkits.axes_grid1 import make_axes_locatable
13 13 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
14 14
15 15 from schainpy.model.proc.jroproc_base import Operation
16 16 from schainpy.utils import log
17 17
18 18 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
19 19 blu_values = matplotlib.pyplot.get_cmap(
20 20 'seismic_r', 20)(numpy.arange(20))[10:15]
21 21 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
22 22 'jro', numpy.vstack((blu_values, jet_values)))
23 23 matplotlib.pyplot.register_cmap(cmap=ncmap)
24 24
25 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'RdBu_r', 'seismic')]
25 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis', 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
26 26
27 27
28 28 def figpause(interval):
29 29 backend = plt.rcParams['backend']
30 30 if backend in matplotlib.rcsetup.interactive_bk:
31 31 figManager = matplotlib._pylab_helpers.Gcf.get_active()
32 32 if figManager is not None:
33 33 canvas = figManager.canvas
34 34 if canvas.figure.stale:
35 35 canvas.draw()
36 36 canvas.start_event_loop(interval)
37 37 return
38 38
39 39
40 40 class PlotData(Operation, Process):
41 41 '''
42 42 Base class for Schain plotting operations
43 43 '''
44 44
45 45 CODE = 'Figure'
46 46 colormap = 'jro'
47 47 bgcolor = 'white'
48 CONFLATE = False
49 __MAXNUMX = 80
48 CONFLATE = False
50 49 __missing = 1E30
51 50
52 51 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
53 52 'zlimits', 'xlabel', 'ylabel', 'xaxis','cb_label', 'title',
54 53 'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',
55 54 'showprofile', 'decimation']
56 55
57 56 def __init__(self, **kwargs):
58 57
59 58 Operation.__init__(self, plot=True, **kwargs)
60 59 Process.__init__(self)
61 60
62 61 self.kwargs['code'] = self.CODE
63 62 self.mp = False
64 63 self.data = None
65 64 self.isConfig = False
66 65 self.figures = []
67 66 self.axes = []
68 67 self.cb_axes = []
69 68 self.localtime = kwargs.pop('localtime', True)
70 69 self.show = kwargs.get('show', True)
71 70 self.save = kwargs.get('save', False)
72 71 self.colormap = kwargs.get('colormap', self.colormap)
73 72 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
74 73 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
75 74 self.colormaps = kwargs.get('colormaps', None)
76 75 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
77 76 self.showprofile = kwargs.get('showprofile', False)
78 77 self.title = kwargs.get('wintitle', self.CODE.upper())
79 78 self.cb_label = kwargs.get('cb_label', None)
80 79 self.cb_labels = kwargs.get('cb_labels', None)
81 80 self.xaxis = kwargs.get('xaxis', 'frequency')
82 81 self.zmin = kwargs.get('zmin', None)
83 82 self.zmax = kwargs.get('zmax', None)
84 83 self.zlimits = kwargs.get('zlimits', None)
85 84 self.xmin = kwargs.get('xmin', None)
86 85 self.xmax = kwargs.get('xmax', None)
87 86 self.xrange = kwargs.get('xrange', 24)
88 87 self.ymin = kwargs.get('ymin', None)
89 88 self.ymax = kwargs.get('ymax', None)
90 89 self.xlabel = kwargs.get('xlabel', None)
91 self.__MAXNUMY = kwargs.get('decimation', 200)
90 self.decimation = kwargs.get('decimation', None)
92 91 self.showSNR = kwargs.get('showSNR', False)
93 92 self.oneFigure = kwargs.get('oneFigure', True)
94 93 self.width = kwargs.get('width', None)
95 94 self.height = kwargs.get('height', None)
96 95 self.colorbar = kwargs.get('colorbar', True)
97 96 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
98 97 self.titles = kwargs.get('titles', [])
99 98 self.polar = False
100 99
101 100 def __fmtTime(self, x, pos):
102 101 '''
103 102 '''
104 103
105 104 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
106 105
107 106 def __setup(self):
108 107 '''
109 108 Common setup for all figures, here figures and axes are created
110 109 '''
111 110
112 111 if self.CODE not in self.data:
113 112 raise ValueError(log.error('Missing data for {}'.format(self.CODE),
114 113 self.name))
115 114
116 115 self.setup()
117 116
118 117 self.time_label = 'LT' if self.localtime else 'UTC'
119 118 if self.data.localtime:
120 119 self.getDateTime = datetime.datetime.fromtimestamp
121 120 else:
122 121 self.getDateTime = datetime.datetime.utcfromtimestamp
123 122
124 123 if self.width is None:
125 124 self.width = 8
126 125
127 126 self.figures = []
128 127 self.axes = []
129 128 self.cb_axes = []
130 129 self.pf_axes = []
131 130 self.cmaps = []
132 131
133 132 size = '15%' if self.ncols == 1 else '30%'
134 133 pad = '4%' if self.ncols == 1 else '8%'
135 134
136 135 if self.oneFigure:
137 136 if self.height is None:
138 137 self.height = 1.4 * self.nrows + 1
139 138 fig = plt.figure(figsize=(self.width, self.height),
140 139 edgecolor='k',
141 140 facecolor='w')
142 141 self.figures.append(fig)
143 142 for n in range(self.nplots):
144 143 ax = fig.add_subplot(self.nrows, self.ncols,
145 144 n + 1, polar=self.polar)
146 145 ax.tick_params(labelsize=8)
147 146 ax.firsttime = True
148 147 ax.index = 0
149 148 ax.press = None
150 149 self.axes.append(ax)
151 150 if self.showprofile:
152 151 cax = self.__add_axes(ax, size=size, pad=pad)
153 152 cax.tick_params(labelsize=8)
154 153 self.pf_axes.append(cax)
155 154 else:
156 155 if self.height is None:
157 156 self.height = 3
158 157 for n in range(self.nplots):
159 158 fig = plt.figure(figsize=(self.width, self.height),
160 159 edgecolor='k',
161 160 facecolor='w')
162 161 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
163 162 ax.tick_params(labelsize=8)
164 163 ax.firsttime = True
165 164 ax.index = 0
166 165 ax.press = None
167 166 self.figures.append(fig)
168 167 self.axes.append(ax)
169 168 if self.showprofile:
170 169 cax = self.__add_axes(ax, size=size, pad=pad)
171 170 cax.tick_params(labelsize=8)
172 171 self.pf_axes.append(cax)
173 172
174 173 for n in range(self.nrows):
175 174 if self.colormaps is not None:
176 175 cmap = plt.get_cmap(self.colormaps[n])
177 176 else:
178 177 cmap = plt.get_cmap(self.colormap)
179 178 cmap.set_bad(self.bgcolor, 1.)
180 179 self.cmaps.append(cmap)
181 180
182 181 for fig in self.figures:
183 182 fig.canvas.mpl_connect('key_press_event', self.OnKeyPress)
184 183 fig.canvas.mpl_connect('scroll_event', self.OnBtnScroll)
185 184 fig.canvas.mpl_connect('button_press_event', self.onBtnPress)
186 185 fig.canvas.mpl_connect('motion_notify_event', self.onMotion)
187 186 fig.canvas.mpl_connect('button_release_event', self.onBtnRelease)
188 187 if self.show:
189 188 fig.show()
190 189
191 190 def OnKeyPress(self, event):
192 191 '''
193 192 Event for pressing keys (up, down) change colormap
194 193 '''
195 194 ax = event.inaxes
196 195 if ax in self.axes:
197 196 if event.key == 'down':
198 197 ax.index += 1
199 198 elif event.key == 'up':
200 199 ax.index -= 1
201 200 if ax.index < 0:
202 201 ax.index = len(CMAPS) - 1
203 202 elif ax.index == len(CMAPS):
204 203 ax.index = 0
205 204 cmap = CMAPS[ax.index]
206 205 ax.cbar.set_cmap(cmap)
207 206 ax.cbar.draw_all()
208 207 ax.plt.set_cmap(cmap)
209 208 ax.cbar.patch.figure.canvas.draw()
210 209 self.colormap = cmap.name
211 210
212 211 def OnBtnScroll(self, event):
213 212 '''
214 213 Event for scrolling, scale figure
215 214 '''
216 215 cb_ax = event.inaxes
217 216 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
218 217 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
219 218 pt = ax.cbar.ax.bbox.get_points()[:, 1]
220 219 nrm = ax.cbar.norm
221 220 vmin, vmax, p0, p1, pS = (
222 221 nrm.vmin, nrm.vmax, pt[0], pt[1], event.y)
223 222 scale = 2 if event.step == 1 else 0.5
224 223 point = vmin + (vmax - vmin) / (p1 - p0) * (pS - p0)
225 224 ax.cbar.norm.vmin = point - scale * (point - vmin)
226 225 ax.cbar.norm.vmax = point - scale * (point - vmax)
227 226 ax.plt.set_norm(ax.cbar.norm)
228 227 ax.cbar.draw_all()
229 228 ax.cbar.patch.figure.canvas.draw()
230 229
231 230 def onBtnPress(self, event):
232 231 '''
233 232 Event for mouse button press
234 233 '''
235 234 cb_ax = event.inaxes
236 235 if cb_ax is None:
237 236 return
238 237
239 238 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
240 239 cb_ax.press = event.x, event.y
241 240 else:
242 241 cb_ax.press = None
243 242
244 243 def onMotion(self, event):
245 244 '''
246 245 Event for move inside colorbar
247 246 '''
248 247 cb_ax = event.inaxes
249 248 if cb_ax is None:
250 249 return
251 250 if cb_ax not in [ax.cbar.ax for ax in self.axes if ax.cbar]:
252 251 return
253 252 if cb_ax.press is None:
254 253 return
255 254
256 255 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
257 256 xprev, yprev = cb_ax.press
258 257 dx = event.x - xprev
259 258 dy = event.y - yprev
260 259 cb_ax.press = event.x, event.y
261 260 scale = ax.cbar.norm.vmax - ax.cbar.norm.vmin
262 261 perc = 0.03
263 262
264 263 if event.button == 1:
265 264 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
266 265 ax.cbar.norm.vmax -= (perc * scale) * numpy.sign(dy)
267 266 elif event.button == 3:
268 267 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
269 268 ax.cbar.norm.vmax += (perc * scale) * numpy.sign(dy)
270 269
271 270 ax.cbar.draw_all()
272 271 ax.plt.set_norm(ax.cbar.norm)
273 272 ax.cbar.patch.figure.canvas.draw()
274 273
275 274 def onBtnRelease(self, event):
276 275 '''
277 276 Event for mouse button release
278 277 '''
279 278 cb_ax = event.inaxes
280 279 if cb_ax is not None:
281 280 cb_ax.press = None
282 281
283 282 def __add_axes(self, ax, size='30%', pad='8%'):
284 283 '''
285 284 Add new axes to the given figure
286 285 '''
287 286 divider = make_axes_locatable(ax)
288 287 nax = divider.new_horizontal(size=size, pad=pad)
289 288 ax.figure.add_axes(nax)
290 289 return nax
291 290
292 291 self.setup()
293 292
294 293 def setup(self):
295 294 '''
296 295 This method should be implemented in the child class, the following
297 296 attributes should be set:
298 297
299 298 self.nrows: number of rows
300 299 self.ncols: number of cols
301 300 self.nplots: number of plots (channels or pairs)
302 301 self.ylabel: label for Y axes
303 302 self.titles: list of axes title
304 303
305 304 '''
306 305 raise(NotImplementedError, 'Implement this method in child class')
307 306
308 307 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
309 308 '''
310 309 Create a masked array for missing data
311 310 '''
312 311 if x_buffer.shape[0] < 2:
313 312 return x_buffer, y_buffer, z_buffer
314 313
315 314 deltas = x_buffer[1:] - x_buffer[0:-1]
316 315 x_median = numpy.median(deltas)
317 316
318 317 index = numpy.where(deltas > 5 * x_median)
319 318
320 319 if len(index[0]) != 0:
321 320 z_buffer[::, index[0], ::] = self.__missing
322 321 z_buffer = numpy.ma.masked_inside(z_buffer,
323 322 0.99 * self.__missing,
324 323 1.01 * self.__missing)
325 324
326 325 return x_buffer, y_buffer, z_buffer
327 326
328 327 def decimate(self):
329 328
330 329 # dx = int(len(self.x)/self.__MAXNUMX) + 1
331 dy = int(len(self.y) / self.__MAXNUMY) + 1
330 dy = int(len(self.y) / self.decimation) + 1
332 331
333 332 # x = self.x[::dx]
334 333 x = self.x
335 334 y = self.y[::dy]
336 335 z = self.z[::, ::, ::dy]
337 336
338 337 return x, y, z
339 338
340 339 def format(self):
341 340 '''
342 341 Set min and max values, labels, ticks and titles
343 342 '''
344 343
345 344 if self.xmin is None:
346 345 xmin = self.min_time
347 346 else:
348 347 if self.xaxis is 'time':
349 348 dt = self.getDateTime(self.min_time)
350 349 xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) -
351 350 datetime.datetime(1970, 1, 1)).total_seconds()
352 351 if self.data.localtime:
353 352 xmin += time.timezone
354 353 else:
355 354 xmin = self.xmin
356 355
357 356 if self.xmax is None:
358 357 xmax = xmin + self.xrange * 60 * 60
359 358 else:
360 359 if self.xaxis is 'time':
361 360 dt = self.getDateTime(self.max_time)
362 361 xmax = (dt.replace(hour=int(self.xmax), minute=59, second=59) -
363 362 datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=1)).total_seconds()
364 363 if self.data.localtime:
365 364 xmax += time.timezone
366 365 else:
367 366 xmax = self.xmax
368 367
369 368 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
370 369 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
371 370
372 Y = numpy.array([10, 20, 50, 100, 200, 500, 1000, 2000])
371 Y = numpy.array([5, 10, 20, 50, 100, 200, 500, 1000, 2000])
373 372 i = 1 if numpy.where(ymax-ymin < Y)[0][0] < 0 else numpy.where(ymax-ymin < Y)[0][0]
374 373 ystep = Y[i] / 5
375 374
376 375 for n, ax in enumerate(self.axes):
377 376 if ax.firsttime:
378 377 ax.set_facecolor(self.bgcolor)
379 378 ax.yaxis.set_major_locator(MultipleLocator(ystep))
380 379 if self.xaxis is 'time':
381 380 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
382 381 ax.xaxis.set_major_locator(LinearLocator(9))
383 382 if self.xlabel is not None:
384 383 ax.set_xlabel(self.xlabel)
385 384 ax.set_ylabel(self.ylabel)
386 385 ax.firsttime = False
387 386 if self.showprofile:
388 387 self.pf_axes[n].set_ylim(ymin, ymax)
389 388 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
390 389 self.pf_axes[n].set_xlabel('dB')
391 390 self.pf_axes[n].grid(b=True, axis='x')
392 391 [tick.set_visible(False)
393 392 for tick in self.pf_axes[n].get_yticklabels()]
394 393 if self.colorbar:
395 394 ax.cbar = plt.colorbar(
396 395 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
397 396 ax.cbar.ax.tick_params(labelsize=8)
398 397 ax.cbar.ax.press = None
399 398 if self.cb_label:
400 399 ax.cbar.set_label(self.cb_label, size=8)
401 400 elif self.cb_labels:
402 401 ax.cbar.set_label(self.cb_labels[n], size=8)
403 402 else:
404 403 ax.cbar = None
405 404
406 405 if not self.polar:
407 406 ax.set_xlim(xmin, xmax)
408 407 ax.set_ylim(ymin, ymax)
409 408 ax.set_title('{} - {} {}'.format(
410 409 self.titles[n],
411 410 self.getDateTime(self.max_time).strftime('%H:%M:%S'),
412 411 self.time_label),
413 412 size=8)
414 413 else:
415 414 ax.set_title('{}'.format(self.titles[n]), size=8)
416 415 ax.set_ylim(0, 90)
417 416 ax.set_yticks(numpy.arange(0, 90, 20))
418 417 ax.yaxis.labelpad = 40
419 418
420 419 def __plot(self):
421 420 '''
422 421 '''
423 422 log.success('Plotting', self.name)
424 423
425 self.plot()
426 self.format()
424 try:
425 self.plot()
426 self.format()
427 except:
428 log.warning('{} Plot could not be updated... check data'.format(self.CODE), self.name)
427 429
428 430 for n, fig in enumerate(self.figures):
429 431 if self.nrows == 0 or self.nplots == 0:
430 432 log.warning('No data', self.name)
431 433 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
432 434 fig.canvas.manager.set_window_title(self.CODE)
433 435 continue
434 436
435 437 fig.tight_layout()
436 438 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
437 439 self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
438 440 fig.canvas.draw()
439 441
440 442 if self.save and self.data.ended:
441 443 channels = range(self.nrows)
442 444 if self.oneFigure:
443 445 label = ''
444 446 else:
445 447 label = '_{}'.format(channels[n])
446 448 figname = os.path.join(
447 449 self.save,
448 450 '{}{}_{}.png'.format(
449 451 self.CODE,
450 452 label,
451 453 self.getDateTime(self.saveTime).strftime(
452 '%y%m%d_%H%M%S'),
454 '%Y%m%d_%H%M%S'),
453 455 )
454 456 )
455 457 log.log('Saving figure: {}'.format(figname), self.name)
456 458 fig.savefig(figname)
457 459
458 460 def plot(self):
459 461 '''
460 462 '''
461 463 raise(NotImplementedError, 'Implement this method in child class')
462 464
463 465 def run(self):
464 466
465 467 log.success('Starting', self.name)
466 468
467 469 context = zmq.Context()
468 470 receiver = context.socket(zmq.SUB)
469 471 receiver.setsockopt(zmq.SUBSCRIBE, '')
470 472 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
471 473
472 474 if 'server' in self.kwargs['parent']:
473 475 receiver.connect(
474 476 'ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
475 477 else:
476 478 receiver.connect("ipc:///tmp/zmq.plots")
477 479
478 480 while True:
479 481 try:
480 482 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
481 483 if self.data.localtime and self.localtime:
482 484 self.times = self.data.times
483 485 elif self.data.localtime and not self.localtime:
484 486 self.times = self.data.times + time.timezone
485 487 elif not self.data.localtime and self.localtime:
486 488 self.times = self.data.times - time.timezone
487 489 else:
488 490 self.times = self.data.times
489 491
490 492 self.min_time = self.times[0]
491 493 self.max_time = self.times[-1]
492 494
493 495 if self.isConfig is False:
494 496 self.__setup()
495 497 self.isConfig = True
496 498
497 499 self.__plot()
498 500
499 501 except zmq.Again as e:
500 502 log.log('Waiting for data...')
501 503 if self.data:
502 504 figpause(self.data.throttle)
503 505 else:
504 506 time.sleep(2)
505 507
506 508 def close(self):
507 509 if self.data:
508 510 self.__plot()
509 511
510 512
511 513 class PlotSpectraData(PlotData):
512 514 '''
513 515 Plot for Spectra data
514 516 '''
515 517
516 518 CODE = 'spc'
517 519 colormap = 'jro'
518 520
519 521 def setup(self):
520 522 self.nplots = len(self.data.channels)
521 523 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
522 524 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
523 525 self.width = 3.4 * self.ncols
524 526 self.height = 3 * self.nrows
525 527 self.cb_label = 'dB'
526 528 if self.showprofile:
527 529 self.width += 0.8 * self.ncols
528 530
529 self.ylabel = 'Range [Km]'
531 self.ylabel = 'Range [km]'
530 532
531 533 def plot(self):
532 534 if self.xaxis == "frequency":
533 535 x = self.data.xrange[0]
534 536 self.xlabel = "Frequency (kHz)"
535 537 elif self.xaxis == "time":
536 538 x = self.data.xrange[1]
537 539 self.xlabel = "Time (ms)"
538 540 else:
539 541 x = self.data.xrange[2]
540 542 self.xlabel = "Velocity (m/s)"
541 543
542 544 if self.CODE == 'spc_mean':
543 545 x = self.data.xrange[2]
544 546 self.xlabel = "Velocity (m/s)"
545 547
546 548 self.titles = []
547 549
548 550 y = self.data.heights
549 551 self.y = y
550 552 z = self.data['spc']
551 553
552 554 for n, ax in enumerate(self.axes):
553 555 noise = self.data['noise'][n][-1]
554 556 if self.CODE == 'spc_mean':
555 557 mean = self.data['mean'][n][-1]
556 558 if ax.firsttime:
557 559 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
558 560 self.xmin = self.xmin if self.xmin else -self.xmax
559 561 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
560 562 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
561 563 ax.plt = ax.pcolormesh(x, y, z[n].T,
562 564 vmin=self.zmin,
563 565 vmax=self.zmax,
564 566 cmap=plt.get_cmap(self.colormap)
565 567 )
566 568
567 569 if self.showprofile:
568 570 ax.plt_profile = self.pf_axes[n].plot(
569 571 self.data['rti'][n][-1], y)[0]
570 572 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
571 573 color="k", linestyle="dashed", lw=1)[0]
572 574 if self.CODE == 'spc_mean':
573 575 ax.plt_mean = ax.plot(mean, y, color='k')[0]
574 576 else:
575 577 ax.plt.set_array(z[n].T.ravel())
576 578 if self.showprofile:
577 579 ax.plt_profile.set_data(self.data['rti'][n][-1], y)
578 580 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
579 581 if self.CODE == 'spc_mean':
580 582 ax.plt_mean.set_data(mean, y)
581 583
582 584 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
583 585 self.saveTime = self.max_time
584 586
585 587
586 588 class PlotCrossSpectraData(PlotData):
587 589
588 590 CODE = 'cspc'
589 591 zmin_coh = None
590 592 zmax_coh = None
591 593 zmin_phase = None
592 594 zmax_phase = None
593 595
594 596 def setup(self):
595 597
596 598 self.ncols = 4
597 599 self.nrows = len(self.data.pairs)
598 600 self.nplots = self.nrows * 4
599 601 self.width = 3.4 * self.ncols
600 602 self.height = 3 * self.nrows
601 self.ylabel = 'Range [Km]'
603 self.ylabel = 'Range [km]'
602 604 self.showprofile = False
603 605
604 606 def plot(self):
605 607
606 608 if self.xaxis == "frequency":
607 609 x = self.data.xrange[0]
608 610 self.xlabel = "Frequency (kHz)"
609 611 elif self.xaxis == "time":
610 612 x = self.data.xrange[1]
611 613 self.xlabel = "Time (ms)"
612 614 else:
613 615 x = self.data.xrange[2]
614 616 self.xlabel = "Velocity (m/s)"
615 617
616 618 self.titles = []
617 619
618 620 y = self.data.heights
619 621 self.y = y
620 622 spc = self.data['spc']
621 623 cspc = self.data['cspc']
622 624
623 625 for n in range(self.nrows):
624 626 noise = self.data['noise'][n][-1]
625 627 pair = self.data.pairs[n]
626 628 ax = self.axes[4 * n]
627 629 ax3 = self.axes[4 * n + 3]
628 630 if ax.firsttime:
629 631 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
630 632 self.xmin = self.xmin if self.xmin else -self.xmax
631 633 self.zmin = self.zmin if self.zmin else numpy.nanmin(spc)
632 634 self.zmax = self.zmax if self.zmax else numpy.nanmax(spc)
633 635 ax.plt = ax.pcolormesh(x, y, spc[pair[0]].T,
634 636 vmin=self.zmin,
635 637 vmax=self.zmax,
636 638 cmap=plt.get_cmap(self.colormap)
637 639 )
638 640 else:
639 641 ax.plt.set_array(spc[pair[0]].T.ravel())
640 642 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
641 643
642 644 ax = self.axes[4 * n + 1]
643 645 if ax.firsttime:
644 646 ax.plt = ax.pcolormesh(x, y, spc[pair[1]].T,
645 647 vmin=self.zmin,
646 648 vmax=self.zmax,
647 649 cmap=plt.get_cmap(self.colormap)
648 650 )
649 651 else:
650 652 ax.plt.set_array(spc[pair[1]].T.ravel())
651 653 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
652 654
653 655 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
654 656 coh = numpy.abs(out)
655 657 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
656 658
657 659 ax = self.axes[4 * n + 2]
658 660 if ax.firsttime:
659 661 ax.plt = ax.pcolormesh(x, y, coh.T,
660 662 vmin=0,
661 663 vmax=1,
662 664 cmap=plt.get_cmap(self.colormap_coh)
663 665 )
664 666 else:
665 667 ax.plt.set_array(coh.T.ravel())
666 668 self.titles.append(
667 669 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
668 670
669 671 ax = self.axes[4 * n + 3]
670 672 if ax.firsttime:
671 673 ax.plt = ax.pcolormesh(x, y, phase.T,
672 674 vmin=-180,
673 675 vmax=180,
674 676 cmap=plt.get_cmap(self.colormap_phase)
675 677 )
676 678 else:
677 679 ax.plt.set_array(phase.T.ravel())
678 680 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
679 681
680 682 self.saveTime = self.max_time
681 683
682 684
683 685 class PlotSpectraMeanData(PlotSpectraData):
684 686 '''
685 687 Plot for Spectra and Mean
686 688 '''
687 689 CODE = 'spc_mean'
688 690 colormap = 'jro'
689 691
690 692
691 693 class PlotRTIData(PlotData):
692 694 '''
693 695 Plot for RTI data
694 696 '''
695 697
696 698 CODE = 'rti'
697 699 colormap = 'jro'
698 700
699 701 def setup(self):
700 702 self.xaxis = 'time'
701 703 self.ncols = 1
702 704 self.nrows = len(self.data.channels)
703 705 self.nplots = len(self.data.channels)
704 self.ylabel = 'Range [Km]'
706 self.ylabel = 'Range [km]'
705 707 self.cb_label = 'dB'
706 708 self.titles = ['{} Channel {}'.format(
707 709 self.CODE.upper(), x) for x in range(self.nrows)]
708 710
709 711 def plot(self):
710 712 self.x = self.times
711 713 self.y = self.data.heights
712 714 self.z = self.data[self.CODE]
713 715 self.z = numpy.ma.masked_invalid(self.z)
714 716
715 for n, ax in enumerate(self.axes):
717 if self.decimation is None:
718 x, y, z = self.fill_gaps(self.x, self.y, self.z)
719 else:
716 720 x, y, z = self.fill_gaps(*self.decimate())
721
722 for n, ax in enumerate(self.axes):
717 723 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
718 724 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
719 725 if ax.firsttime:
720 726 ax.plt = ax.pcolormesh(x, y, z[n].T,
721 727 vmin=self.zmin,
722 728 vmax=self.zmax,
723 729 cmap=plt.get_cmap(self.colormap)
724 730 )
725 731 if self.showprofile:
726 732 ax.plot_profile = self.pf_axes[n].plot(
727 733 self.data['rti'][n][-1], self.y)[0]
728 734 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
729 735 color="k", linestyle="dashed", lw=1)[0]
730 736 else:
731 737 ax.collections.remove(ax.collections[0])
732 738 ax.plt = ax.pcolormesh(x, y, z[n].T,
733 739 vmin=self.zmin,
734 740 vmax=self.zmax,
735 741 cmap=plt.get_cmap(self.colormap)
736 742 )
737 743 if self.showprofile:
738 744 ax.plot_profile.set_data(self.data['rti'][n][-1], self.y)
739 745 ax.plot_noise.set_data(numpy.repeat(
740 746 self.data['noise'][n][-1], len(self.y)), self.y)
741 747
742 748 self.saveTime = self.min_time
743 749
744 750
745 751 class PlotCOHData(PlotRTIData):
746 752 '''
747 753 Plot for Coherence data
748 754 '''
749 755
750 756 CODE = 'coh'
751 757
752 758 def setup(self):
753 759 self.xaxis = 'time'
754 760 self.ncols = 1
755 761 self.nrows = len(self.data.pairs)
756 762 self.nplots = len(self.data.pairs)
757 self.ylabel = 'Range [Km]'
763 self.ylabel = 'Range [km]'
758 764 if self.CODE == 'coh':
759 765 self.cb_label = ''
760 766 self.titles = [
761 767 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
762 768 else:
763 769 self.cb_label = 'Degrees'
764 770 self.titles = [
765 771 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
766 772
767 773
768 774 class PlotPHASEData(PlotCOHData):
769 775 '''
770 776 Plot for Phase map data
771 777 '''
772 778
773 779 CODE = 'phase'
774 780 colormap = 'seismic'
775 781
776 782
777 783 class PlotNoiseData(PlotData):
778 784 '''
779 785 Plot for noise
780 786 '''
781 787
782 788 CODE = 'noise'
783 789
784 790 def setup(self):
785 791 self.xaxis = 'time'
786 792 self.ncols = 1
787 793 self.nrows = 1
788 794 self.nplots = 1
789 795 self.ylabel = 'Intensity [dB]'
790 796 self.titles = ['Noise']
791 797 self.colorbar = False
792 798
793 799 def plot(self):
794 800
795 801 x = self.times
796 802 xmin = self.min_time
797 803 xmax = xmin + self.xrange * 60 * 60
798 804 Y = self.data[self.CODE]
799 805
800 806 if self.axes[0].firsttime:
801 807 for ch in self.data.channels:
802 808 y = Y[ch]
803 809 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
804 810 plt.legend()
805 811 else:
806 812 for ch in self.data.channels:
807 813 y = Y[ch]
808 814 self.axes[0].lines[ch].set_data(x, y)
809 815
810 816 self.ymin = numpy.nanmin(Y) - 5
811 817 self.ymax = numpy.nanmax(Y) + 5
812 818 self.saveTime = self.min_time
813 819
814 820
815 821 class PlotSNRData(PlotRTIData):
816 822 '''
817 823 Plot for SNR Data
818 824 '''
819 825
820 826 CODE = 'snr'
821 827 colormap = 'jet'
822 828
823 829
824 830 class PlotDOPData(PlotRTIData):
825 831 '''
826 832 Plot for DOPPLER Data
827 833 '''
828 834
829 835 CODE = 'dop'
830 836 colormap = 'jet'
831 837
832 838
833 839 class PlotSkyMapData(PlotData):
834 840 '''
835 841 Plot for meteors detection data
836 842 '''
837 843
838 844 CODE = 'param'
839 845
840 846 def setup(self):
841 847
842 848 self.ncols = 1
843 849 self.nrows = 1
844 850 self.width = 7.2
845 851 self.height = 7.2
846 852 self.nplots = 1
847 853 self.xlabel = 'Zonal Zenith Angle (deg)'
848 854 self.ylabel = 'Meridional Zenith Angle (deg)'
849 855 self.polar = True
850 856 self.ymin = -180
851 857 self.ymax = 180
852 858 self.colorbar = False
853 859
854 860 def plot(self):
855 861
856 862 arrayParameters = numpy.concatenate(self.data['param'])
857 863 error = arrayParameters[:, -1]
858 864 indValid = numpy.where(error == 0)[0]
859 865 finalMeteor = arrayParameters[indValid, :]
860 866 finalAzimuth = finalMeteor[:, 3]
861 867 finalZenith = finalMeteor[:, 4]
862 868
863 869 x = finalAzimuth * numpy.pi / 180
864 870 y = finalZenith
865 871
866 872 ax = self.axes[0]
867 873
868 874 if ax.firsttime:
869 875 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
870 876 else:
871 877 ax.plot.set_data(x, y)
872 878
873 879 dt1 = self.getDateTime(self.min_time).strftime('%y/%m/%d %H:%M:%S')
874 880 dt2 = self.getDateTime(self.max_time).strftime('%y/%m/%d %H:%M:%S')
875 881 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
876 882 dt2,
877 883 len(x))
878 884 self.titles[0] = title
879 885 self.saveTime = self.max_time
880 886
881 887
882 888 class PlotParamData(PlotRTIData):
883 889 '''
884 890 Plot for data_param object
885 891 '''
886 892
887 893 CODE = 'param'
888 894 colormap = 'seismic'
889 895
890 896 def setup(self):
891 897 self.xaxis = 'time'
892 898 self.ncols = 1
893 899 self.nrows = self.data.shape(self.CODE)[0]
894 900 self.nplots = self.nrows
895 901 if self.showSNR:
896 902 self.nrows += 1
897 903 self.nplots += 1
898 904
899 self.ylabel = 'Height [Km]'
905 self.ylabel = 'Height [km]'
900 906 if not self.titles:
901 907 self.titles = self.data.parameters \
902 908 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
903 909 if self.showSNR:
904 910 self.titles.append('SNR')
905 911
906 912 def plot(self):
907 913 self.data.normalize_heights()
908 914 self.x = self.times
909 915 self.y = self.data.heights
910 916 if self.showSNR:
911 917 self.z = numpy.concatenate(
912 918 (self.data[self.CODE], self.data['snr'])
913 919 )
914 920 else:
915 921 self.z = self.data[self.CODE]
916 922
917 923 self.z = numpy.ma.masked_invalid(self.z)
918 924
919 for n, ax in enumerate(self.axes):
920
925 if self.decimation is None:
926 x, y, z = self.fill_gaps(self.x, self.y, self.z)
927 else:
921 928 x, y, z = self.fill_gaps(*self.decimate())
929
930 for n, ax in enumerate(self.axes):
931
922 932 self.zmax = self.zmax if self.zmax is not None else numpy.max(
923 933 self.z[n])
924 934 self.zmin = self.zmin if self.zmin is not None else numpy.min(
925 935 self.z[n])
926 936
927 937 if ax.firsttime:
928 938 if self.zlimits is not None:
929 939 self.zmin, self.zmax = self.zlimits[n]
930 940
931 941 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
932 942 vmin=self.zmin,
933 943 vmax=self.zmax,
934 944 cmap=self.cmaps[n]
935 945 )
936 946 else:
937 947 if self.zlimits is not None:
938 948 self.zmin, self.zmax = self.zlimits[n]
939 949 ax.collections.remove(ax.collections[0])
940 950 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
941 951 vmin=self.zmin,
942 952 vmax=self.zmax,
943 953 cmap=self.cmaps[n]
944 954 )
945 955
946 956 self.saveTime = self.min_time
947 957
948 958
949 959 class PlotOutputData(PlotParamData):
950 960 '''
951 961 Plot data_output object
952 962 '''
953 963
954 964 CODE = 'output'
955 965 colormap = 'seismic'
Show file before | Show file after | Hide comments
@@ -1,495 +1,501
1 import numpy
2 import datetime
1 import os
3 2 import sys
3 import datetime
4 import numpy
4 5 import matplotlib
5 6
6 if 'linux' in sys.platform:
7 if 'BACKEND' in os.environ:
8 matplotlib.use(os.environ['BACKEND'])
9 elif 'linux' in sys.platform:
7 10 matplotlib.use("TkAgg")
8
9 if 'darwin' in sys.platform:
10 matplotlib.use('TKAgg')
11 elif 'darwin' in sys.platform:
12 matplotlib.use('TkAgg')
13 else:
14 from schainpy.utils import log
15 log.warning('Using default Backend="Agg"', 'INFO')
16 matplotlib.use('Agg')
11 17 # Qt4Agg', 'GTK', 'GTKAgg', 'ps', 'agg', 'cairo', 'MacOSX', 'GTKCairo', 'WXAgg', 'template', 'TkAgg', 'GTK3Cairo', 'GTK3Agg', 'svg', 'WebAgg', 'CocoaAgg', 'emf', 'gdk', 'WX'
12 18 import matplotlib.pyplot
13 19
14 20 from mpl_toolkits.axes_grid1 import make_axes_locatable
15 21 from matplotlib.ticker import FuncFormatter, LinearLocator
16 22
17 23 ###########################################
18 24 # Actualizacion de las funciones del driver
19 25 ###########################################
20 26
21 27 # create jro colormap
22 28
23 29 jet_values = matplotlib.pyplot.get_cmap("jet", 100)(numpy.arange(100))[10:90]
24 30 blu_values = matplotlib.pyplot.get_cmap(
25 31 "seismic_r", 20)(numpy.arange(20))[10:15]
26 32 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
27 33 "jro", numpy.vstack((blu_values, jet_values)))
28 34 matplotlib.pyplot.register_cmap(cmap=ncmap)
29 35
30 36
31 37 def createFigure(id, wintitle, width, height, facecolor="w", show=True, dpi=80):
32 38
33 39 matplotlib.pyplot.ioff()
34 40
35 41 fig = matplotlib.pyplot.figure(num=id, facecolor=facecolor, figsize=(
36 42 1.0 * width / dpi, 1.0 * height / dpi))
37 43 fig.canvas.manager.set_window_title(wintitle)
38 44 # fig.canvas.manager.resize(width, height)
39 45 matplotlib.pyplot.ion()
40 46
41 47 if show:
42 48 matplotlib.pyplot.show()
43 49
44 50 return fig
45 51
46 52
47 53 def closeFigure(show=False, fig=None):
48 54
49 55 # matplotlib.pyplot.ioff()
50 56 # matplotlib.pyplot.pause(0)
51 57
52 58 if show:
53 59 matplotlib.pyplot.show()
54 60
55 61 if fig != None:
56 62 matplotlib.pyplot.close(fig)
57 63 # matplotlib.pyplot.pause(0)
58 64 # matplotlib.pyplot.ion()
59 65
60 66 return
61 67
62 68 matplotlib.pyplot.close("all")
63 69 # matplotlib.pyplot.pause(0)
64 70 # matplotlib.pyplot.ion()
65 71
66 72 return
67 73
68 74
69 75 def saveFigure(fig, filename):
70 76
71 77 # matplotlib.pyplot.ioff()
72 78 fig.savefig(filename, dpi=matplotlib.pyplot.gcf().dpi)
73 79 # matplotlib.pyplot.ion()
74 80
75 81
76 82 def clearFigure(fig):
77 83
78 84 fig.clf()
79 85
80 86
81 87 def setWinTitle(fig, title):
82 88
83 89 fig.canvas.manager.set_window_title(title)
84 90
85 91
86 92 def setTitle(fig, title):
87 93
88 94 fig.suptitle(title)
89 95
90 96
91 97 def createAxes(fig, nrow, ncol, xpos, ypos, colspan, rowspan, polar=False):
92 98
93 99 matplotlib.pyplot.ioff()
94 100 matplotlib.pyplot.figure(fig.number)
95 101 axes = matplotlib.pyplot.subplot2grid((nrow, ncol),
96 102 (xpos, ypos),
97 103 colspan=colspan,
98 104 rowspan=rowspan,
99 105 polar=polar)
100 106
101 107 matplotlib.pyplot.ion()
102 108 return axes
103 109
104 110
105 111 def setAxesText(ax, text):
106 112
107 113 ax.annotate(text,
108 114 xy=(.1, .99),
109 115 xycoords='figure fraction',
110 116 horizontalalignment='left',
111 117 verticalalignment='top',
112 118 fontsize=10)
113 119
114 120
115 121 def printLabels(ax, xlabel, ylabel, title):
116 122
117 123 ax.set_xlabel(xlabel, size=11)
118 124 ax.set_ylabel(ylabel, size=11)
119 125 ax.set_title(title, size=8)
120 126
121 127
122 128 def createPline(ax, x, y, xmin, xmax, ymin, ymax, xlabel='', ylabel='', title='',
123 129 ticksize=9, xtick_visible=True, ytick_visible=True,
124 130 nxticks=4, nyticks=10,
125 131 grid=None, color='blue'):
126 132 """
127 133
128 134 Input:
129 135 grid : None, 'both', 'x', 'y'
130 136 """
131 137
132 138 matplotlib.pyplot.ioff()
133 139
134 140 ax.set_xlim([xmin, xmax])
135 141 ax.set_ylim([ymin, ymax])
136 142
137 143 printLabels(ax, xlabel, ylabel, title)
138 144
139 145 ######################################################
140 146 if (xmax - xmin) <= 1:
141 147 xtickspos = numpy.linspace(xmin, xmax, nxticks)
142 148 xtickspos = numpy.array([float("%.1f" % i) for i in xtickspos])
143 149 ax.set_xticks(xtickspos)
144 150 else:
145 151 xtickspos = numpy.arange(nxticks) * \
146 152 int((xmax - xmin) / (nxticks)) + int(xmin)
147 153 # xtickspos = numpy.arange(nxticks)*float(xmax-xmin)/float(nxticks) + int(xmin)
148 154 ax.set_xticks(xtickspos)
149 155
150 156 for tick in ax.get_xticklabels():
151 157 tick.set_visible(xtick_visible)
152 158
153 159 for tick in ax.xaxis.get_major_ticks():
154 160 tick.label.set_fontsize(ticksize)
155 161
156 162 ######################################################
157 163 for tick in ax.get_yticklabels():
158 164 tick.set_visible(ytick_visible)
159 165
160 166 for tick in ax.yaxis.get_major_ticks():
161 167 tick.label.set_fontsize(ticksize)
162 168
163 169 ax.plot(x, y, color=color)
164 170 iplot = ax.lines[-1]
165 171
166 172 ######################################################
167 173 if '0.' in matplotlib.__version__[0:2]:
168 174 print "The matplotlib version has to be updated to 1.1 or newer"
169 175 return iplot
170 176
171 177 if '1.0.' in matplotlib.__version__[0:4]:
172 178 print "The matplotlib version has to be updated to 1.1 or newer"
173 179 return iplot
174 180
175 181 if grid != None:
176 182 ax.grid(b=True, which='major', axis=grid)
177 183
178 184 matplotlib.pyplot.tight_layout()
179 185
180 186 matplotlib.pyplot.ion()
181 187
182 188 return iplot
183 189
184 190
185 191 def set_linedata(ax, x, y, idline):
186 192
187 193 ax.lines[idline].set_data(x, y)
188 194
189 195
190 196 def pline(iplot, x, y, xlabel='', ylabel='', title=''):
191 197
192 198 ax = iplot.axes
193 199
194 200 printLabels(ax, xlabel, ylabel, title)
195 201
196 202 set_linedata(ax, x, y, idline=0)
197 203
198 204
199 205 def addpline(ax, x, y, color, linestyle, lw):
200 206
201 207 ax.plot(x, y, color=color, linestyle=linestyle, lw=lw)
202 208
203 209
204 210 def createPcolor(ax, x, y, z, xmin, xmax, ymin, ymax, zmin, zmax,
205 211 xlabel='', ylabel='', title='', ticksize=9,
206 212 colormap='jet', cblabel='', cbsize="5%",
207 213 XAxisAsTime=False):
208 214
209 215 matplotlib.pyplot.ioff()
210 216
211 217 divider = make_axes_locatable(ax)
212 218 ax_cb = divider.new_horizontal(size=cbsize, pad=0.05)
213 219 fig = ax.get_figure()
214 220 fig.add_axes(ax_cb)
215 221
216 222 ax.set_xlim([xmin, xmax])
217 223 ax.set_ylim([ymin, ymax])
218 224
219 225 printLabels(ax, xlabel, ylabel, title)
220 226
221 227 z = numpy.ma.masked_invalid(z)
222 228 cmap = matplotlib.pyplot.get_cmap(colormap)
223 229 cmap.set_bad('black', 1.)
224 230 imesh = ax.pcolormesh(x, y, z.T, vmin=zmin, vmax=zmax, cmap=cmap)
225 231 cb = matplotlib.pyplot.colorbar(imesh, cax=ax_cb)
226 232 cb.set_label(cblabel)
227 233
228 234 # for tl in ax_cb.get_yticklabels():
229 235 # tl.set_visible(True)
230 236
231 237 for tick in ax.yaxis.get_major_ticks():
232 238 tick.label.set_fontsize(ticksize)
233 239
234 240 for tick in ax.xaxis.get_major_ticks():
235 241 tick.label.set_fontsize(ticksize)
236 242
237 243 for tick in cb.ax.get_yticklabels():
238 244 tick.set_fontsize(ticksize)
239 245
240 246 ax_cb.yaxis.tick_right()
241 247
242 248 if '0.' in matplotlib.__version__[0:2]:
243 249 print "The matplotlib version has to be updated to 1.1 or newer"
244 250 return imesh
245 251
246 252 if '1.0.' in matplotlib.__version__[0:4]:
247 253 print "The matplotlib version has to be updated to 1.1 or newer"
248 254 return imesh
249 255
250 256 matplotlib.pyplot.tight_layout()
251 257
252 258 if XAxisAsTime:
253 259
254 260 def func(x, pos): return ('%s') % (
255 261 datetime.datetime.utcfromtimestamp(x).strftime("%H:%M:%S"))
256 262 ax.xaxis.set_major_formatter(FuncFormatter(func))
257 263 ax.xaxis.set_major_locator(LinearLocator(7))
258 264
259 265 matplotlib.pyplot.ion()
260 266 return imesh
261 267
262 268
263 269 def pcolor(imesh, z, xlabel='', ylabel='', title=''):
264 270
265 271 z = z.T
266 272 ax = imesh.axes
267 273 printLabels(ax, xlabel, ylabel, title)
268 274 imesh.set_array(z.ravel())
269 275
270 276
271 277 def addpcolor(ax, x, y, z, zmin, zmax, xlabel='', ylabel='', title='', colormap='jet'):
272 278
273 279 printLabels(ax, xlabel, ylabel, title)
274 280
275 281 ax.pcolormesh(x, y, z.T, vmin=zmin, vmax=zmax,
276 282 cmap=matplotlib.pyplot.get_cmap(colormap))
277 283
278 284
279 285 def addpcolorbuffer(ax, x, y, z, zmin, zmax, xlabel='', ylabel='', title='', colormap='jet'):
280 286
281 287 printLabels(ax, xlabel, ylabel, title)
282 288
283 289 ax.collections.remove(ax.collections[0])
284 290
285 291 z = numpy.ma.masked_invalid(z)
286 292
287 293 cmap = matplotlib.pyplot.get_cmap(colormap)
288 294 cmap.set_bad('black', 1.)
289 295
290 296 ax.pcolormesh(x, y, z.T, vmin=zmin, vmax=zmax, cmap=cmap)
291 297
292 298
293 299 def createPmultiline(ax, x, y, xmin, xmax, ymin, ymax, xlabel='', ylabel='', title='', legendlabels=None,
294 300 ticksize=9, xtick_visible=True, ytick_visible=True,
295 301 nxticks=4, nyticks=10,
296 302 grid=None):
297 303 """
298 304
299 305 Input:
300 306 grid : None, 'both', 'x', 'y'
301 307 """
302 308
303 309 matplotlib.pyplot.ioff()
304 310
305 311 lines = ax.plot(x.T, y)
306 312 leg = ax.legend(lines, legendlabels, loc='upper right')
307 313 leg.get_frame().set_alpha(0.5)
308 314 ax.set_xlim([xmin, xmax])
309 315 ax.set_ylim([ymin, ymax])
310 316 printLabels(ax, xlabel, ylabel, title)
311 317
312 318 xtickspos = numpy.arange(nxticks) * \
313 319 int((xmax - xmin) / (nxticks)) + int(xmin)
314 320 ax.set_xticks(xtickspos)
315 321
316 322 for tick in ax.get_xticklabels():
317 323 tick.set_visible(xtick_visible)
318 324
319 325 for tick in ax.xaxis.get_major_ticks():
320 326 tick.label.set_fontsize(ticksize)
321 327
322 328 for tick in ax.get_yticklabels():
323 329 tick.set_visible(ytick_visible)
324 330
325 331 for tick in ax.yaxis.get_major_ticks():
326 332 tick.label.set_fontsize(ticksize)
327 333
328 334 iplot = ax.lines[-1]
329 335
330 336 if '0.' in matplotlib.__version__[0:2]:
331 337 print "The matplotlib version has to be updated to 1.1 or newer"
332 338 return iplot
333 339
334 340 if '1.0.' in matplotlib.__version__[0:4]:
335 341 print "The matplotlib version has to be updated to 1.1 or newer"
336 342 return iplot
337 343
338 344 if grid != None:
339 345 ax.grid(b=True, which='major', axis=grid)
340 346
341 347 matplotlib.pyplot.tight_layout()
342 348
343 349 matplotlib.pyplot.ion()
344 350
345 351 return iplot
346 352
347 353
348 354 def pmultiline(iplot, x, y, xlabel='', ylabel='', title=''):
349 355
350 356 ax = iplot.axes
351 357
352 358 printLabels(ax, xlabel, ylabel, title)
353 359
354 360 for i in range(len(ax.lines)):
355 361 line = ax.lines[i]
356 362 line.set_data(x[i, :], y)
357 363
358 364
359 365 def createPmultilineYAxis(ax, x, y, xmin, xmax, ymin, ymax, xlabel='', ylabel='', title='', legendlabels=None,
360 366 ticksize=9, xtick_visible=True, ytick_visible=True,
361 367 nxticks=4, nyticks=10, marker='.', markersize=10, linestyle="None",
362 368 grid=None, XAxisAsTime=False):
363 369 """
364 370
365 371 Input:
366 372 grid : None, 'both', 'x', 'y'
367 373 """
368 374
369 375 matplotlib.pyplot.ioff()
370 376
371 377 # lines = ax.plot(x, y.T, marker=marker,markersize=markersize,linestyle=linestyle)
372 378 lines = ax.plot(x, y.T)
373 379 # leg = ax.legend(lines, legendlabels, loc=2, bbox_to_anchor=(1.01, 1.00), numpoints=1, handlelength=1.5, \
374 380 # handletextpad=0.5, borderpad=0.5, labelspacing=0.5, borderaxespad=0.)
375 381
376 382 leg = ax.legend(lines, legendlabels,
377 383 loc='upper right', bbox_to_anchor=(1.16, 1), borderaxespad=0)
378 384
379 385 for label in leg.get_texts():
380 386 label.set_fontsize(9)
381 387
382 388 ax.set_xlim([xmin, xmax])
383 389 ax.set_ylim([ymin, ymax])
384 390 printLabels(ax, xlabel, ylabel, title)
385 391
386 392 # xtickspos = numpy.arange(nxticks)*int((xmax-xmin)/(nxticks)) + int(xmin)
387 393 # ax.set_xticks(xtickspos)
388 394
389 395 for tick in ax.get_xticklabels():
390 396 tick.set_visible(xtick_visible)
391 397
392 398 for tick in ax.xaxis.get_major_ticks():
393 399 tick.label.set_fontsize(ticksize)
394 400
395 401 for tick in ax.get_yticklabels():
396 402 tick.set_visible(ytick_visible)
397 403
398 404 for tick in ax.yaxis.get_major_ticks():
399 405 tick.label.set_fontsize(ticksize)
400 406
401 407 iplot = ax.lines[-1]
402 408
403 409 if '0.' in matplotlib.__version__[0:2]:
404 410 print "The matplotlib version has to be updated to 1.1 or newer"
405 411 return iplot
406 412
407 413 if '1.0.' in matplotlib.__version__[0:4]:
408 414 print "The matplotlib version has to be updated to 1.1 or newer"
409 415 return iplot
410 416
411 417 if grid != None:
412 418 ax.grid(b=True, which='major', axis=grid)
413 419
414 420 matplotlib.pyplot.tight_layout()
415 421
416 422 if XAxisAsTime:
417 423
418 424 def func(x, pos): return ('%s') % (
419 425 datetime.datetime.utcfromtimestamp(x).strftime("%H:%M:%S"))
420 426 ax.xaxis.set_major_formatter(FuncFormatter(func))
421 427 ax.xaxis.set_major_locator(LinearLocator(7))
422 428
423 429 matplotlib.pyplot.ion()
424 430
425 431 return iplot
426 432
427 433
428 434 def pmultilineyaxis(iplot, x, y, xlabel='', ylabel='', title=''):
429 435
430 436 ax = iplot.axes
431 437
432 438 printLabels(ax, xlabel, ylabel, title)
433 439
434 440 for i in range(len(ax.lines)):
435 441 line = ax.lines[i]
436 442 line.set_data(x, y[i, :])
437 443
438 444
439 445 def createPolar(ax, x, y,
440 446 xlabel='', ylabel='', title='', ticksize=9,
441 447 colormap='jet', cblabel='', cbsize="5%",
442 448 XAxisAsTime=False):
443 449
444 450 matplotlib.pyplot.ioff()
445 451
446 452 ax.plot(x, y, 'bo', markersize=5)
447 453 # ax.set_rmax(90)
448 454 ax.set_ylim(0, 90)
449 455 ax.set_yticks(numpy.arange(0, 90, 20))
450 456 # ax.text(0, -110, ylabel, rotation='vertical', va ='center', ha = 'center' ,size='11')
451 457 # ax.text(0, 50, ylabel, rotation='vertical', va ='center', ha = 'left' ,size='11')
452 458 # ax.text(100, 100, 'example', ha='left', va='center', rotation='vertical')
453 459 ax.yaxis.labelpad = 40
454 460 printLabels(ax, xlabel, ylabel, title)
455 461 iplot = ax.lines[-1]
456 462
457 463 if '0.' in matplotlib.__version__[0:2]:
458 464 print "The matplotlib version has to be updated to 1.1 or newer"
459 465 return iplot
460 466
461 467 if '1.0.' in matplotlib.__version__[0:4]:
462 468 print "The matplotlib version has to be updated to 1.1 or newer"
463 469 return iplot
464 470
465 471 # if grid != None:
466 472 # ax.grid(b=True, which='major', axis=grid)
467 473
468 474 matplotlib.pyplot.tight_layout()
469 475
470 476 matplotlib.pyplot.ion()
471 477
472 478 return iplot
473 479
474 480
475 481 def polar(iplot, x, y, xlabel='', ylabel='', title=''):
476 482
477 483 ax = iplot.axes
478 484
479 485 # ax.text(0, -110, ylabel, rotation='vertical', va ='center', ha = 'center',size='11')
480 486 printLabels(ax, xlabel, ylabel, title)
481 487
482 488 set_linedata(ax, x, y, idline=0)
483 489
484 490
485 491 def draw(fig):
486 492
487 493 if type(fig) == 'int':
488 494 raise ValueError, "Error drawing: Fig parameter should be a matplotlib figure object figure"
489 495
490 496 fig.canvas.draw()
491 497
492 498
493 499 def pause(interval=0.000001):
494 500
495 501 matplotlib.pyplot.pause(interval)
Show file before | Show file after | Hide comments
@@ -1,1833 +1,1824
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time
14 14 import datetime
15 15 import traceback
16 16 import zmq
17 17
18 18 try:
19 19 from gevent import sleep
20 20 except:
21 21 from time import sleep
22 22
23 23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
24 24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
25 from schainpy.utils import log
25 26
26 27 LOCALTIME = True
27 28
28 29
29 30 def isNumber(cad):
30 31 """
31 32 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
32 33
33 34 Excepciones:
34 35 Si un determinado string no puede ser convertido a numero
35 36 Input:
36 37 str, string al cual se le analiza para determinar si convertible a un numero o no
37 38
38 39 Return:
39 40 True : si el string es uno numerico
40 41 False : no es un string numerico
41 42 """
42 43 try:
43 44 float(cad)
44 45 return True
45 46 except:
46 47 return False
47 48
48 49
49 50 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
50 51 """
51 52 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
52 53
53 54 Inputs:
54 55 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
55 56
56 57 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
57 58 segundos contados desde 01/01/1970.
58 59 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
59 60 segundos contados desde 01/01/1970.
60 61
61 62 Return:
62 63 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
63 64 fecha especificado, de lo contrario retorna False.
64 65
65 66 Excepciones:
66 67 Si el archivo no existe o no puede ser abierto
67 68 Si la cabecera no puede ser leida.
68 69
69 70 """
70 71 basicHeaderObj = BasicHeader(LOCALTIME)
71 72
72 73 try:
73 74 fp = open(filename, 'rb')
74 75 except IOError:
75 76 print "The file %s can't be opened" % (filename)
76 77 return 0
77 78
78 79 sts = basicHeaderObj.read(fp)
79 80 fp.close()
80 81
81 82 if not(sts):
82 83 print "Skipping the file %s because it has not a valid header" % (filename)
83 84 return 0
84 85
85 86 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
86 87 return 0
87 88
88 89 return 1
89 90
90 91
91 92 def isTimeInRange(thisTime, startTime, endTime):
92 93 if endTime >= startTime:
93 94 if (thisTime < startTime) or (thisTime > endTime):
94 95 return 0
95 96 return 1
96 97 else:
97 98 if (thisTime < startTime) and (thisTime > endTime):
98 99 return 0
99 100 return 1
100 101
101 102
102 103 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
103 104 """
104 105 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
105 106
106 107 Inputs:
107 108 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
108 109
109 110 startDate : fecha inicial del rango seleccionado en formato datetime.date
110 111
111 112 endDate : fecha final del rango seleccionado en formato datetime.date
112 113
113 114 startTime : tiempo inicial del rango seleccionado en formato datetime.time
114 115
115 116 endTime : tiempo final del rango seleccionado en formato datetime.time
116 117
117 118 Return:
118 119 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
119 120 fecha especificado, de lo contrario retorna False.
120 121
121 122 Excepciones:
122 123 Si el archivo no existe o no puede ser abierto
123 124 Si la cabecera no puede ser leida.
124 125
125 126 """
126 127
127 128 try:
128 129 fp = open(filename, 'rb')
129 130 except IOError:
130 131 print "The file %s can't be opened" % (filename)
131 132 return None
132 133
133 134 firstBasicHeaderObj = BasicHeader(LOCALTIME)
134 135 systemHeaderObj = SystemHeader()
135 136 radarControllerHeaderObj = RadarControllerHeader()
136 137 processingHeaderObj = ProcessingHeader()
137 138
138 139 lastBasicHeaderObj = BasicHeader(LOCALTIME)
139 140
140 141 sts = firstBasicHeaderObj.read(fp)
141 142
142 143 if not(sts):
143 144 print "[Reading] Skipping the file %s because it has not a valid header" % (filename)
144 145 return None
145 146
146 147 if not systemHeaderObj.read(fp):
147 148 return None
148 149
149 150 if not radarControllerHeaderObj.read(fp):
150 151 return None
151 152
152 153 if not processingHeaderObj.read(fp):
153 154 return None
154 155
155 156 filesize = os.path.getsize(filename)
156 157
157 158 offset = processingHeaderObj.blockSize + 24 # header size
158 159
159 160 if filesize <= offset:
160 161 print "[Reading] %s: This file has not enough data" % filename
161 162 return None
162 163
163 164 fp.seek(-offset, 2)
164 165
165 166 sts = lastBasicHeaderObj.read(fp)
166 167
167 168 fp.close()
168 169
169 170 thisDatetime = lastBasicHeaderObj.datatime
170 171 thisTime_last_block = thisDatetime.time()
171 172
172 173 thisDatetime = firstBasicHeaderObj.datatime
173 174 thisDate = thisDatetime.date()
174 175 thisTime_first_block = thisDatetime.time()
175 176
176 177 # General case
177 178 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
178 179 #-----------o----------------------------o-----------
179 180 # startTime endTime
180 181
181 182 if endTime >= startTime:
182 183 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
183 184 return None
184 185
185 186 return thisDatetime
186 187
187 188 # If endTime < startTime then endTime belongs to the next day
188 189
189 190 #<<<<<<<<<<<o o>>>>>>>>>>>
190 191 #-----------o----------------------------o-----------
191 192 # endTime startTime
192 193
193 194 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 195 return None
195 196
196 197 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 198 return None
198 199
199 200 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 201 return None
201 202
202 203 return thisDatetime
203 204
204 205
205 206 def isFolderInDateRange(folder, startDate=None, endDate=None):
206 207 """
207 208 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
208 209
209 210 Inputs:
210 211 folder : nombre completo del directorio.
211 212 Su formato deberia ser "/path_root/?YYYYDDD"
212 213
213 214 siendo:
214 215 YYYY : Anio (ejemplo 2015)
215 216 DDD : Dia del anio (ejemplo 305)
216 217
217 218 startDate : fecha inicial del rango seleccionado en formato datetime.date
218 219
219 220 endDate : fecha final del rango seleccionado en formato datetime.date
220 221
221 222 Return:
222 223 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
223 224 fecha especificado, de lo contrario retorna False.
224 225 Excepciones:
225 226 Si el directorio no tiene el formato adecuado
226 227 """
227 228
228 229 basename = os.path.basename(folder)
229 230
230 231 if not isRadarFolder(basename):
231 232 print "The folder %s has not the rigth format" % folder
232 233 return 0
233 234
234 235 if startDate and endDate:
235 236 thisDate = getDateFromRadarFolder(basename)
236 237
237 238 if thisDate < startDate:
238 239 return 0
239 240
240 241 if thisDate > endDate:
241 242 return 0
242 243
243 244 return 1
244 245
245 246
246 247 def isFileInDateRange(filename, startDate=None, endDate=None):
247 248 """
248 249 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
249 250
250 251 Inputs:
251 252 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
252 253
253 254 Su formato deberia ser "?YYYYDDDsss"
254 255
255 256 siendo:
256 257 YYYY : Anio (ejemplo 2015)
257 258 DDD : Dia del anio (ejemplo 305)
258 259 sss : set
259 260
260 261 startDate : fecha inicial del rango seleccionado en formato datetime.date
261 262
262 263 endDate : fecha final del rango seleccionado en formato datetime.date
263 264
264 265 Return:
265 266 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
266 267 fecha especificado, de lo contrario retorna False.
267 268 Excepciones:
268 269 Si el archivo no tiene el formato adecuado
269 270 """
270 271
271 272 basename = os.path.basename(filename)
272 273
273 274 if not isRadarFile(basename):
274 275 print "The filename %s has not the rigth format" % filename
275 276 return 0
276 277
277 278 if startDate and endDate:
278 279 thisDate = getDateFromRadarFile(basename)
279 280
280 281 if thisDate < startDate:
281 282 return 0
282 283
283 284 if thisDate > endDate:
284 285 return 0
285 286
286 287 return 1
287 288
288 289
289 290 def getFileFromSet(path, ext, set):
290 291 validFilelist = []
291 292 fileList = os.listdir(path)
292 293
293 294 # 0 1234 567 89A BCDE
294 295 # H YYYY DDD SSS .ext
295 296
296 297 for thisFile in fileList:
297 298 try:
298 299 year = int(thisFile[1:5])
299 300 doy = int(thisFile[5:8])
300 301 except:
301 302 continue
302 303
303 304 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
304 305 continue
305 306
306 307 validFilelist.append(thisFile)
307 308
308 309 myfile = fnmatch.filter(
309 310 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
310 311
311 312 if len(myfile) != 0:
312 313 return myfile[0]
313 314 else:
314 315 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
315 316 print 'the filename %s does not exist' % filename
316 317 print '...going to the last file: '
317 318
318 319 if validFilelist:
319 320 validFilelist = sorted(validFilelist, key=str.lower)
320 321 return validFilelist[-1]
321 322
322 323 return None
323 324
324 325
325 326 def getlastFileFromPath(path, ext):
326 327 """
327 328 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
328 329 al final de la depuracion devuelve el ultimo file de la lista que quedo.
329 330
330 331 Input:
331 332 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
332 333 ext : extension de los files contenidos en una carpeta
333 334
334 335 Return:
335 336 El ultimo file de una determinada carpeta, no se considera el path.
336 337 """
337 338 validFilelist = []
338 339 fileList = os.listdir(path)
339 340
340 341 # 0 1234 567 89A BCDE
341 342 # H YYYY DDD SSS .ext
342 343
343 344 for thisFile in fileList:
344 345
345 346 year = thisFile[1:5]
346 347 if not isNumber(year):
347 348 continue
348 349
349 350 doy = thisFile[5:8]
350 351 if not isNumber(doy):
351 352 continue
352 353
353 354 year = int(year)
354 355 doy = int(doy)
355 356
356 357 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
357 358 continue
358 359
359 360 validFilelist.append(thisFile)
360 361
361 362 if validFilelist:
362 363 validFilelist = sorted(validFilelist, key=str.lower)
363 364 return validFilelist[-1]
364 365
365 366 return None
366 367
367 368
368 369 def checkForRealPath(path, foldercounter, year, doy, set, ext):
369 370 """
370 371 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
371 372 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
372 373 el path exacto de un determinado file.
373 374
374 375 Example :
375 376 nombre correcto del file es .../.../D2009307/P2009307367.ext
376 377
377 378 Entonces la funcion prueba con las siguientes combinaciones
378 379 .../.../y2009307367.ext
379 380 .../.../Y2009307367.ext
380 381 .../.../x2009307/y2009307367.ext
381 382 .../.../x2009307/Y2009307367.ext
382 383 .../.../X2009307/y2009307367.ext
383 384 .../.../X2009307/Y2009307367.ext
384 385 siendo para este caso, la ultima combinacion de letras, identica al file buscado
385 386
386 387 Return:
387 388 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
388 389 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
389 390 para el filename
390 391 """
391 392 fullfilename = None
392 393 find_flag = False
393 394 filename = None
394 395
395 396 prefixDirList = [None, 'd', 'D']
396 397 if ext.lower() == ".r": # voltage
397 398 prefixFileList = ['d', 'D']
398 399 elif ext.lower() == ".pdata": # spectra
399 400 prefixFileList = ['p', 'P']
400 401 else:
401 402 return None, filename
402 403
403 404 # barrido por las combinaciones posibles
404 405 for prefixDir in prefixDirList:
405 406 thispath = path
406 407 if prefixDir != None:
407 408 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
408 409 if foldercounter == 0:
409 410 thispath = os.path.join(path, "%s%04d%03d" %
410 411 (prefixDir, year, doy))
411 412 else:
412 413 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
413 414 prefixDir, year, doy, foldercounter))
414 415 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
415 416 # formo el nombre del file xYYYYDDDSSS.ext
416 417 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
417 418 fullfilename = os.path.join(
418 419 thispath, filename) # formo el path completo
419 420
420 421 if os.path.exists(fullfilename): # verifico que exista
421 422 find_flag = True
422 423 break
423 424 if find_flag:
424 425 break
425 426
426 427 if not(find_flag):
427 428 return None, filename
428 429
429 430 return fullfilename, filename
430 431
431 432
432 433 def isRadarFolder(folder):
433 434 try:
434 435 year = int(folder[1:5])
435 436 doy = int(folder[5:8])
436 437 except:
437 438 return 0
438 439
439 440 return 1
440 441
441 442
442 443 def isRadarFile(file):
443 try:
444 try:
444 445 year = int(file[1:5])
445 446 doy = int(file[5:8])
446 447 set = int(file[8:11])
447 448 except:
448 449 return 0
449 450
450 451 return 1
451 452
452 453
453 454 def getDateFromRadarFile(file):
454 try:
455 try:
455 456 year = int(file[1:5])
456 457 doy = int(file[5:8])
457 set = int(file[8:11])
458 set = int(file[8:11])
458 459 except:
459 460 return None
460 461
461 462 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
462 463 return thisDate
463 464
464 465
465 466 def getDateFromRadarFolder(folder):
466 467 try:
467 468 year = int(folder[1:5])
468 469 doy = int(folder[5:8])
469 470 except:
470 471 return None
471 472
472 473 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
473 474 return thisDate
474 475
475 476
476 477 class JRODataIO:
477 478
478 479 c = 3E8
479 480
480 481 isConfig = False
481 482
482 483 basicHeaderObj = None
483 484
484 485 systemHeaderObj = None
485 486
486 487 radarControllerHeaderObj = None
487 488
488 489 processingHeaderObj = None
489 490
490 491 dtype = None
491 492
492 493 pathList = []
493 494
494 495 filenameList = []
495 496
496 497 filename = None
497 498
498 499 ext = None
499 500
500 501 flagIsNewFile = 1
501 502
502 503 flagDiscontinuousBlock = 0
503 504
504 505 flagIsNewBlock = 0
505 506
506 507 fp = None
507 508
508 509 firstHeaderSize = 0
509 510
510 511 basicHeaderSize = 24
511 512
512 513 versionFile = 1103
513 514
514 515 fileSize = None
515 516
516 517 # ippSeconds = None
517 518
518 519 fileSizeByHeader = None
519 520
520 521 fileIndex = None
521 522
522 523 profileIndex = None
523 524
524 525 blockIndex = None
525 526
526 527 nTotalBlocks = None
527 528
528 529 maxTimeStep = 30
529 530
530 531 lastUTTime = None
531 532
532 533 datablock = None
533 534
534 535 dataOut = None
535 536
536 537 blocksize = None
537 538
538 539 getByBlock = False
539 540
540 541 def __init__(self):
541 542
542 543 raise NotImplementedError
543 544
544 545 def run(self):
545 546
546 547 raise NotImplementedError
547 548
548 549 def getDtypeWidth(self):
549 550
550 551 dtype_index = get_dtype_index(self.dtype)
551 552 dtype_width = get_dtype_width(dtype_index)
552 553
553 554 return dtype_width
554 555
555 556 def getAllowedArgs(self):
556 557 if hasattr(self, '__attrs__'):
557 558 return self.__attrs__
558 559 else:
559 560 return inspect.getargspec(self.run).args
560 561
561 562
562 563 class JRODataReader(JRODataIO):
563 564
564 565 online = 0
565 566
566 567 realtime = 0
567 568
568 569 nReadBlocks = 0
569 570
570 571 delay = 10 # number of seconds waiting a new file
571 572
572 573 nTries = 3 # quantity tries
573 574
574 575 nFiles = 3 # number of files for searching
575 576
576 577 path = None
577 578
578 579 foldercounter = 0
579 580
580 581 flagNoMoreFiles = 0
581 582
582 583 datetimeList = []
583 584
584 585 __isFirstTimeOnline = 1
585 586
586 587 __printInfo = True
587 588
588 589 profileIndex = None
589 590
590 591 nTxs = 1
591 592
592 593 txIndex = None
593 594
594 595 # Added--------------------
595 596
596 597 selBlocksize = None
597 598
598 599 selBlocktime = None
599 600
600 601 def __init__(self):
601 602 """
602 603 This class is used to find data files
603 604
604 605 Example:
605 606 reader = JRODataReader()
606 607 fileList = reader.findDataFiles()
607 608
608 609 """
609 610 pass
610 611
611 612 def createObjByDefault(self):
612 613 """
613 614
614 615 """
615 616 raise NotImplementedError
616 617
617 618 def getBlockDimension(self):
618 619
619 620 raise NotImplementedError
620 621
621 622 def searchFilesOffLine(self,
622 623 path,
623 624 startDate=None,
624 625 endDate=None,
625 626 startTime=datetime.time(0, 0, 0),
626 627 endTime=datetime.time(23, 59, 59),
627 628 set=None,
628 629 expLabel='',
629 630 ext='.r',
630 631 cursor=None,
631 632 skip=None,
632 633 walk=True):
633 634
634 635 self.filenameList = []
635 636 self.datetimeList = []
636 637
637 638 pathList = []
638 639
639 640 dateList, pathList = self.findDatafiles(
640 641 path, startDate, endDate, expLabel, ext, walk, include_path=True)
641 642
642 643 if dateList == []:
643 644 return [], []
644 645
645 646 if len(dateList) > 1:
646 647 print "[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList))
647 648 else:
648 649 print "[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0])
649 650
650 651 filenameList = []
651 652 datetimeList = []
652
653
653 654 for thisPath in pathList:
654 655
655 656 fileList = glob.glob1(thisPath, "*%s" % ext)
656 657 fileList.sort()
657 658
658 skippedFileList = []
659
660 if cursor is not None and skip is not None:
661
662 if skip == 0:
663 skippedFileList = []
664 else:
665 skippedFileList = fileList[cursor *
666 skip: cursor * skip + skip]
667
668 else:
669 skippedFileList = fileList
670
671 for file in skippedFileList:
659 for file in fileList:
672 660
673 661 filename = os.path.join(thisPath, file)
674 662
675 663 if not isFileInDateRange(filename, startDate, endDate):
676 664 continue
677 665
678 666 thisDatetime = isFileInTimeRange(
679 667 filename, startDate, endDate, startTime, endTime)
680 668
681 669 if not(thisDatetime):
682 670 continue
683 671
684 672 filenameList.append(filename)
685 673 datetimeList.append(thisDatetime)
686 674
675 if cursor is not None and skip is not None:
676 filenameList = filenameList[cursor * skip:cursor * skip + skip]
677 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
678
687 679 if not(filenameList):
688 680 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path)
689 681 return [], []
690 682
691 print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)
692 print
683 print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)
693 684
694 685 # for i in range(len(filenameList)):
695 686 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
696 687
697 688 self.filenameList = filenameList
698 689 self.datetimeList = datetimeList
699 690
700 691 return pathList, filenameList
701 692
702 693 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
703 694 """
704 695 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
705 696 devuelve el archivo encontrado ademas de otros datos.
706 697
707 698 Input:
708 699 path : carpeta donde estan contenidos los files que contiene data
709 700
710 701 expLabel : Nombre del subexperimento (subfolder)
711 702
712 703 ext : extension de los files
713 704
714 705 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
715 706
716 707 Return:
717 708 directory : eL directorio donde esta el file encontrado
718 709 filename : el ultimo file de una determinada carpeta
719 710 year : el anho
720 711 doy : el numero de dia del anho
721 712 set : el set del archivo
722 713
723 714
724 715 """
725 716 if not os.path.isdir(path):
726 717 return None, None, None, None, None, None
727 718
728 719 dirList = []
729 720
730 721 if not walk:
731 722 fullpath = path
732 723 foldercounter = 0
733 724 else:
734 725 # Filtra solo los directorios
735 726 for thisPath in os.listdir(path):
736 727 if not os.path.isdir(os.path.join(path, thisPath)):
737 728 continue
738 729 if not isRadarFolder(thisPath):
739 730 continue
740 731
741 732 dirList.append(thisPath)
742 733
743 734 if not(dirList):
744 735 return None, None, None, None, None, None
745 736
746 737 dirList = sorted(dirList, key=str.lower)
747 738
748 739 doypath = dirList[-1]
749 740 foldercounter = int(doypath.split('_')[1]) if len(
750 741 doypath.split('_')) > 1 else 0
751 742 fullpath = os.path.join(path, doypath, expLabel)
752 743
753 744 print "[Reading] %s folder was found: " % (fullpath)
754 745
755 746 if set == None:
756 747 filename = getlastFileFromPath(fullpath, ext)
757 748 else:
758 749 filename = getFileFromSet(fullpath, ext, set)
759 750
760 751 if not(filename):
761 752 return None, None, None, None, None, None
762 753
763 754 print "[Reading] %s file was found" % (filename)
764 755
765 756 if not(self.__verifyFile(os.path.join(fullpath, filename))):
766 757 return None, None, None, None, None, None
767 758
768 759 year = int(filename[1:5])
769 760 doy = int(filename[5:8])
770 761 set = int(filename[8:11])
771 762
772 763 return fullpath, foldercounter, filename, year, doy, set
773 764
774 765 def __setNextFileOffline(self):
775 766
776 767 idFile = self.fileIndex
777 768
778 769 while (True):
779 770 idFile += 1
780 771 if not(idFile < len(self.filenameList)):
781 772 self.flagNoMoreFiles = 1
782 773 # print "[Reading] No more Files"
783 774 return 0
784 775
785 776 filename = self.filenameList[idFile]
786 777
787 778 if not(self.__verifyFile(filename)):
788 779 continue
789 780
790 781 fileSize = os.path.getsize(filename)
791 782 fp = open(filename, 'rb')
792 783 break
793 784
794 785 self.flagIsNewFile = 1
795 786 self.fileIndex = idFile
796 787 self.filename = filename
797 788 self.fileSize = fileSize
798 789 self.fp = fp
799 790
800 791 # print "[Reading] Setting the file: %s"%self.filename
801 792
802 793 return 1
803 794
804 795 def __setNextFileOnline(self):
805 796 """
806 797 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
807 798 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
808 799 siguientes.
809 800
810 801 Affected:
811 802 self.flagIsNewFile
812 803 self.filename
813 804 self.fileSize
814 805 self.fp
815 806 self.set
816 807 self.flagNoMoreFiles
817 808
818 809 Return:
819 810 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
820 811 1 : si el file fue abierto con exito y esta listo a ser leido
821 812
822 813 Excepciones:
823 814 Si un determinado file no puede ser abierto
824 815 """
825 816 nFiles = 0
826 817 fileOk_flag = False
827 818 firstTime_flag = True
828 819
829 820 self.set += 1
830 821
831 822 if self.set > 999:
832 823 self.set = 0
833 824 self.foldercounter += 1
834 825
835 826 # busca el 1er file disponible
836 827 fullfilename, filename = checkForRealPath(
837 828 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
838 829 if fullfilename:
839 830 if self.__verifyFile(fullfilename, False):
840 831 fileOk_flag = True
841 832
842 833 # si no encuentra un file entonces espera y vuelve a buscar
843 834 if not(fileOk_flag):
844 835 # busco en los siguientes self.nFiles+1 files posibles
845 836 for nFiles in range(self.nFiles + 1):
846 837
847 838 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
848 839 tries = self.nTries
849 840 else:
850 841 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
851 842
852 843 for nTries in range(tries):
853 844 if firstTime_flag:
854 845 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1)
855 846 sleep(self.delay)
856 847 else:
857 848 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
858 849
859 850 fullfilename, filename = checkForRealPath(
860 851 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
861 852 if fullfilename:
862 853 if self.__verifyFile(fullfilename):
863 854 fileOk_flag = True
864 855 break
865 856
866 857 if fileOk_flag:
867 858 break
868 859
869 860 firstTime_flag = False
870 861
871 862 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
872 863 self.set += 1
873 864
874 865 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
875 866 if nFiles == (self.nFiles - 1):
876 867 self.set = 0
877 868 self.doy += 1
878 869 self.foldercounter = 0
879 870
880 871 if fileOk_flag:
881 872 self.fileSize = os.path.getsize(fullfilename)
882 873 self.filename = fullfilename
883 874 self.flagIsNewFile = 1
884 875 if self.fp != None:
885 876 self.fp.close()
886 877 self.fp = open(fullfilename, 'rb')
887 878 self.flagNoMoreFiles = 0
888 879 # print '[Reading] Setting the file: %s' % fullfilename
889 880 else:
890 881 self.fileSize = 0
891 882 self.filename = None
892 883 self.flagIsNewFile = 0
893 884 self.fp = None
894 885 self.flagNoMoreFiles = 1
895 886 # print '[Reading] No more files to read'
896 887
897 888 return fileOk_flag
898 889
899 890 def setNextFile(self):
900 891 if self.fp != None:
901 892 self.fp.close()
902 893
903 894 if self.online:
904 895 newFile = self.__setNextFileOnline()
905 896 else:
906 897 newFile = self.__setNextFileOffline()
907 898
908 899 if not(newFile):
909 900 print '[Reading] No more files to read'
910 901 return 0
911 902
912 903 if self.verbose:
913 904 print '[Reading] Setting the file: %s' % self.filename
914 905
915 906 self.__readFirstHeader()
916 907 self.nReadBlocks = 0
917 908 return 1
918 909
919 910 def __waitNewBlock(self):
920 911 """
921 912 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
922 913
923 914 Si el modo de lectura es OffLine siempre retorn 0
924 915 """
925 916 if not self.online:
926 917 return 0
927 918
928 919 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
929 920 return 0
930 921
931 922 currentPointer = self.fp.tell()
932 923
933 924 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
934 925
935 926 for nTries in range(self.nTries):
936 927
937 928 self.fp.close()
938 929 self.fp = open(self.filename, 'rb')
939 930 self.fp.seek(currentPointer)
940 931
941 932 self.fileSize = os.path.getsize(self.filename)
942 933 currentSize = self.fileSize - currentPointer
943 934
944 935 if (currentSize >= neededSize):
945 936 self.basicHeaderObj.read(self.fp)
946 937 return 1
947 938
948 939 if self.fileSize == self.fileSizeByHeader:
949 940 # self.flagEoF = True
950 941 return 0
951 942
952 943 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
953 944 sleep(self.delay)
954 945
955 946 return 0
956 947
957 948 def waitDataBlock(self, pointer_location):
958 949
959 950 currentPointer = pointer_location
960 951
961 952 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
962 953
963 954 for nTries in range(self.nTries):
964 955 self.fp.close()
965 956 self.fp = open(self.filename, 'rb')
966 957 self.fp.seek(currentPointer)
967 958
968 959 self.fileSize = os.path.getsize(self.filename)
969 960 currentSize = self.fileSize - currentPointer
970 961
971 962 if (currentSize >= neededSize):
972 963 return 1
973 964
974 965 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
975 966 sleep(self.delay)
976 967
977 968 return 0
978 969
979 970 def __jumpToLastBlock(self):
980 971
981 972 if not(self.__isFirstTimeOnline):
982 973 return
983 974
984 975 csize = self.fileSize - self.fp.tell()
985 976 blocksize = self.processingHeaderObj.blockSize
986 977
987 978 # salta el primer bloque de datos
988 979 if csize > self.processingHeaderObj.blockSize:
989 980 self.fp.seek(self.fp.tell() + blocksize)
990 981 else:
991 982 return
992 983
993 984 csize = self.fileSize - self.fp.tell()
994 985 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
995 986 while True:
996 987
997 988 if self.fp.tell() < self.fileSize:
998 989 self.fp.seek(self.fp.tell() + neededsize)
999 990 else:
1000 991 self.fp.seek(self.fp.tell() - neededsize)
1001 992 break
1002 993
1003 994 # csize = self.fileSize - self.fp.tell()
1004 995 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1005 996 # factor = int(csize/neededsize)
1006 997 # if factor > 0:
1007 998 # self.fp.seek(self.fp.tell() + factor*neededsize)
1008 999
1009 1000 self.flagIsNewFile = 0
1010 1001 self.__isFirstTimeOnline = 0
1011 1002
1012 1003 def __setNewBlock(self):
1013 1004 # if self.server is None:
1014 1005 if self.fp == None:
1015 1006 return 0
1016 1007
1017 1008 # if self.online:
1018 1009 # self.__jumpToLastBlock()
1019 1010
1020 1011 if self.flagIsNewFile:
1021 1012 self.lastUTTime = self.basicHeaderObj.utc
1022 1013 return 1
1023 1014
1024 1015 if self.realtime:
1025 1016 self.flagDiscontinuousBlock = 1
1026 1017 if not(self.setNextFile()):
1027 1018 return 0
1028 1019 else:
1029 1020 return 1
1030 1021 # if self.server is None:
1031 1022 currentSize = self.fileSize - self.fp.tell()
1032 1023 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1033 1024 if (currentSize >= neededSize):
1034 1025 self.basicHeaderObj.read(self.fp)
1035 1026 self.lastUTTime = self.basicHeaderObj.utc
1036 1027 return 1
1037 1028 # else:
1038 1029 # self.basicHeaderObj.read(self.zHeader)
1039 1030 # self.lastUTTime = self.basicHeaderObj.utc
1040 1031 # return 1
1041 1032 if self.__waitNewBlock():
1042 1033 self.lastUTTime = self.basicHeaderObj.utc
1043 1034 return 1
1044 1035 # if self.server is None:
1045 1036 if not(self.setNextFile()):
1046 1037 return 0
1047 1038
1048 1039 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1049 1040 self.lastUTTime = self.basicHeaderObj.utc
1050 1041
1051 1042 self.flagDiscontinuousBlock = 0
1052 1043
1053 1044 if deltaTime > self.maxTimeStep:
1054 1045 self.flagDiscontinuousBlock = 1
1055 1046
1056 1047 return 1
1057 1048
1058 1049 def readNextBlock(self):
1059 1050
1060 1051 # Skip block out of startTime and endTime
1061 1052 while True:
1062 1053 if not(self.__setNewBlock()):
1063 1054 return 0
1064 1055
1065 1056 if not(self.readBlock()):
1066 1057 return 0
1067 1058
1068 1059 self.getBasicHeader()
1069 1060 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1070 1061 print "[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1071 1062 self.processingHeaderObj.dataBlocksPerFile,
1072 1063 self.dataOut.datatime.ctime())
1073 1064 continue
1074 1065
1075 1066 break
1076 1067
1077 1068 if self.verbose:
1078 1069 print "[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1079 1070 self.processingHeaderObj.dataBlocksPerFile,
1080 1071 self.dataOut.datatime.ctime())
1081 1072 return 1
1082 1073
1083 1074 def __readFirstHeader(self):
1084 1075
1085 1076 self.basicHeaderObj.read(self.fp)
1086 1077 self.systemHeaderObj.read(self.fp)
1087 1078 self.radarControllerHeaderObj.read(self.fp)
1088 1079 self.processingHeaderObj.read(self.fp)
1089 1080
1090 1081 self.firstHeaderSize = self.basicHeaderObj.size
1091 1082
1092 1083 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1093 1084 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1094 1085 if datatype == 0:
1095 1086 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1096 1087 elif datatype == 1:
1097 1088 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1098 1089 elif datatype == 2:
1099 1090 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1100 1091 elif datatype == 3:
1101 1092 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1102 1093 elif datatype == 4:
1103 1094 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1104 1095 elif datatype == 5:
1105 1096 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1106 1097 else:
1107 1098 raise ValueError, 'Data type was not defined'
1108 1099
1109 1100 self.dtype = datatype_str
1110 1101 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1111 1102 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1112 1103 self.firstHeaderSize + self.basicHeaderSize * \
1113 1104 (self.processingHeaderObj.dataBlocksPerFile - 1)
1114 1105 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1115 1106 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1116 1107 self.getBlockDimension()
1117 1108
1118 1109 def __verifyFile(self, filename, msgFlag=True):
1119 1110
1120 1111 msg = None
1121 1112
1122 1113 try:
1123 1114 fp = open(filename, 'rb')
1124 1115 except IOError:
1125 1116
1126 1117 if msgFlag:
1127 1118 print "[Reading] File %s can't be opened" % (filename)
1128 1119
1129 1120 return False
1130 1121
1131 1122 currentPosition = fp.tell()
1132 1123 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1133 1124
1134 1125 if neededSize == 0:
1135 1126 basicHeaderObj = BasicHeader(LOCALTIME)
1136 1127 systemHeaderObj = SystemHeader()
1137 1128 radarControllerHeaderObj = RadarControllerHeader()
1138 1129 processingHeaderObj = ProcessingHeader()
1139 1130
1140 1131 if not(basicHeaderObj.read(fp)):
1141 1132 fp.close()
1142 1133 return False
1143 1134
1144 1135 if not(systemHeaderObj.read(fp)):
1145 1136 fp.close()
1146 1137 return False
1147 1138
1148 1139 if not(radarControllerHeaderObj.read(fp)):
1149 1140 fp.close()
1150 1141 return False
1151 1142
1152 1143 if not(processingHeaderObj.read(fp)):
1153 1144 fp.close()
1154 1145 return False
1155 1146
1156 1147 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1157 1148 else:
1158 1149 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1159 1150
1160 1151 fp.close()
1161 1152
1162 1153 fileSize = os.path.getsize(filename)
1163 1154 currentSize = fileSize - currentPosition
1164 1155
1165 1156 if currentSize < neededSize:
1166 1157 if msgFlag and (msg != None):
1167 1158 print msg
1168 1159 return False
1169 1160
1170 1161 return True
1171 1162
1172 1163 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1173 1164
1174 1165 path_empty = True
1175 1166
1176 1167 dateList = []
1177 1168 pathList = []
1178 1169
1179 1170 multi_path = path.split(',')
1180 1171
1181 1172 if not walk:
1182 1173
1183 1174 for single_path in multi_path:
1184 1175
1185 1176 if not os.path.isdir(single_path):
1186 1177 continue
1187 1178
1188 1179 fileList = glob.glob1(single_path, "*" + ext)
1189 1180
1190 1181 if not fileList:
1191 1182 continue
1192 1183
1193 1184 path_empty = False
1194 1185
1195 1186 fileList.sort()
1196 1187
1197 1188 for thisFile in fileList:
1198 1189
1199 1190 if not os.path.isfile(os.path.join(single_path, thisFile)):
1200 1191 continue
1201 1192
1202 1193 if not isRadarFile(thisFile):
1203 1194 continue
1204 1195
1205 1196 if not isFileInDateRange(thisFile, startDate, endDate):
1206 1197 continue
1207 1198
1208 1199 thisDate = getDateFromRadarFile(thisFile)
1209 1200
1210 1201 if thisDate in dateList:
1211 1202 continue
1212 1203
1213 1204 dateList.append(thisDate)
1214 1205 pathList.append(single_path)
1215 1206
1216 1207 else:
1217 1208 for single_path in multi_path:
1218 1209
1219 1210 if not os.path.isdir(single_path):
1220 1211 continue
1221 1212
1222 1213 dirList = []
1223 1214
1224 1215 for thisPath in os.listdir(single_path):
1225 1216
1226 1217 if not os.path.isdir(os.path.join(single_path, thisPath)):
1227 1218 continue
1228 1219
1229 1220 if not isRadarFolder(thisPath):
1230 1221 continue
1231 1222
1232 1223 if not isFolderInDateRange(thisPath, startDate, endDate):
1233 1224 continue
1234 1225
1235 1226 dirList.append(thisPath)
1236 1227
1237 1228 if not dirList:
1238 1229 continue
1239 1230
1240 1231 dirList.sort()
1241 1232
1242 1233 for thisDir in dirList:
1243 1234
1244 1235 datapath = os.path.join(single_path, thisDir, expLabel)
1245 1236 fileList = glob.glob1(datapath, "*" + ext)
1246 1237
1247 1238 if not fileList:
1248 1239 continue
1249 1240
1250 1241 path_empty = False
1251 1242
1252 1243 thisDate = getDateFromRadarFolder(thisDir)
1253 1244
1254 1245 pathList.append(datapath)
1255 1246 dateList.append(thisDate)
1256 1247
1257 1248 dateList.sort()
1258 1249
1259 1250 if walk:
1260 1251 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1261 1252 else:
1262 1253 pattern_path = multi_path[0]
1263 1254
1264 1255 if path_empty:
1265 1256 print "[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate)
1266 1257 else:
1267 1258 if not dateList:
1268 1259 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path)
1269 1260
1270 1261 if include_path:
1271 1262 return dateList, pathList
1272 1263
1273 1264 return dateList
1274 1265
1275 1266 def setup(self,
1276 1267 path=None,
1277 1268 startDate=None,
1278 1269 endDate=None,
1279 1270 startTime=datetime.time(0, 0, 0),
1280 1271 endTime=datetime.time(23, 59, 59),
1281 1272 set=None,
1282 1273 expLabel="",
1283 1274 ext=None,
1284 1275 online=False,
1285 1276 delay=60,
1286 1277 walk=True,
1287 1278 getblock=False,
1288 1279 nTxs=1,
1289 1280 realtime=False,
1290 1281 blocksize=None,
1291 1282 blocktime=None,
1292 1283 skip=None,
1293 1284 cursor=None,
1294 1285 warnings=True,
1295 1286 verbose=True,
1296 1287 server=None,
1297 1288 format=None,
1298 1289 oneDDict=None,
1299 1290 twoDDict=None,
1300 1291 ind2DList=None):
1301 1292 if server is not None:
1302 1293 if 'tcp://' in server:
1303 1294 address = server
1304 1295 else:
1305 1296 address = 'ipc:///tmp/%s' % server
1306 1297 self.server = address
1307 1298 self.context = zmq.Context()
1308 1299 self.receiver = self.context.socket(zmq.PULL)
1309 1300 self.receiver.connect(self.server)
1310 1301 time.sleep(0.5)
1311 1302 print '[Starting] ReceiverData from {}'.format(self.server)
1312 1303 else:
1313 1304 self.server = None
1314 1305 if path == None:
1315 1306 raise ValueError, "[Reading] The path is not valid"
1316 1307
1317 1308 if ext == None:
1318 1309 ext = self.ext
1319 1310
1320 1311 if online:
1321 1312 print "[Reading] Searching files in online mode..."
1322 1313
1323 1314 for nTries in range(self.nTries):
1324 1315 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1325 1316 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1326 1317
1327 1318 if fullpath:
1328 1319 break
1329 1320
1330 1321 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1)
1331 1322 sleep(self.delay)
1332 1323
1333 1324 if not(fullpath):
1334 1325 print "[Reading] There 'isn't any valid file in %s" % path
1335 1326 return
1336 1327
1337 1328 self.year = year
1338 1329 self.doy = doy
1339 1330 self.set = set - 1
1340 1331 self.path = path
1341 1332 self.foldercounter = foldercounter
1342 1333 last_set = None
1343 1334 else:
1344 1335 print "[Reading] Searching files in offline mode ..."
1345 1336 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1346 1337 startTime=startTime, endTime=endTime,
1347 1338 set=set, expLabel=expLabel, ext=ext,
1348 1339 walk=walk, cursor=cursor,
1349 1340 skip=skip)
1350 1341
1351 1342 if not(pathList):
1352 1343 self.fileIndex = -1
1353 1344 self.pathList = []
1354 1345 self.filenameList = []
1355 1346 return
1356 1347
1357 1348 self.fileIndex = -1
1358 1349 self.pathList = pathList
1359 1350 self.filenameList = filenameList
1360 1351 file_name = os.path.basename(filenameList[-1])
1361 1352 basename, ext = os.path.splitext(file_name)
1362 1353 last_set = int(basename[-3:])
1363 1354
1364 1355 self.online = online
1365 1356 self.realtime = realtime
1366 1357 self.delay = delay
1367 1358 ext = ext.lower()
1368 1359 self.ext = ext
1369 1360 self.getByBlock = getblock
1370 1361 self.nTxs = nTxs
1371 1362 self.startTime = startTime
1372 1363 self.endTime = endTime
1373 1364 self.endDate = endDate
1374 1365 self.startDate = startDate
1375 1366 # Added-----------------
1376 1367 self.selBlocksize = blocksize
1377 1368 self.selBlocktime = blocktime
1378 1369
1379 1370 # Verbose-----------
1380 1371 self.verbose = verbose
1381 1372 self.warnings = warnings
1382 1373
1383 1374 if not(self.setNextFile()):
1384 1375 if (startDate != None) and (endDate != None):
1385 1376 print "[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime())
1386 1377 elif startDate != None:
1387 1378 print "[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime())
1388 1379 else:
1389 1380 print "[Reading] No files"
1390 1381
1391 1382 self.fileIndex = -1
1392 1383 self.pathList = []
1393 1384 self.filenameList = []
1394 1385 return
1395 1386
1396 1387 # self.getBasicHeader()
1397 1388
1398 1389 if last_set != None:
1399 1390 self.dataOut.last_block = last_set * \
1400 1391 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1401 1392 return
1402 1393
1403 1394 def getBasicHeader(self):
1404 1395
1405 1396 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1406 1397 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1407 1398
1408 1399 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1409 1400
1410 1401 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1411 1402
1412 1403 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1413 1404
1414 1405 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1415 1406
1416 1407 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1417 1408
1418 1409 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1419 1410
1420 1411 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1421 1412
1422 1413 def getFirstHeader(self):
1423 1414
1424 1415 raise NotImplementedError
1425 1416
1426 1417 def getData(self):
1427 1418
1428 1419 raise NotImplementedError
1429 1420
1430 1421 def hasNotDataInBuffer(self):
1431 1422
1432 1423 raise NotImplementedError
1433 1424
1434 1425 def readBlock(self):
1435 1426
1436 1427 raise NotImplementedError
1437 1428
1438 1429 def isEndProcess(self):
1439 1430
1440 1431 return self.flagNoMoreFiles
1441 1432
1442 1433 def printReadBlocks(self):
1443 1434
1444 1435 print "[Reading] Number of read blocks per file %04d" % self.nReadBlocks
1445 1436
1446 1437 def printTotalBlocks(self):
1447 1438
1448 1439 print "[Reading] Number of read blocks %04d" % self.nTotalBlocks
1449 1440
1450 1441 def printNumberOfBlock(self):
1451 1442 'SPAM!'
1452 1443
1453 1444 # if self.flagIsNewBlock:
1454 1445 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1455 1446 # self.processingHeaderObj.dataBlocksPerFile,
1456 1447 # self.dataOut.datatime.ctime())
1457 1448
1458 1449 def printInfo(self):
1459 1450
1460 1451 if self.__printInfo == False:
1461 1452 return
1462 1453
1463 1454 self.basicHeaderObj.printInfo()
1464 1455 self.systemHeaderObj.printInfo()
1465 1456 self.radarControllerHeaderObj.printInfo()
1466 1457 self.processingHeaderObj.printInfo()
1467 1458
1468 1459 self.__printInfo = False
1469 1460
1470 1461 def run(self,
1471 1462 path=None,
1472 1463 startDate=None,
1473 1464 endDate=None,
1474 1465 startTime=datetime.time(0, 0, 0),
1475 1466 endTime=datetime.time(23, 59, 59),
1476 1467 set=None,
1477 1468 expLabel="",
1478 1469 ext=None,
1479 1470 online=False,
1480 1471 delay=60,
1481 1472 walk=True,
1482 1473 getblock=False,
1483 1474 nTxs=1,
1484 1475 realtime=False,
1485 1476 blocksize=None,
1486 1477 blocktime=None,
1487 1478 skip=None,
1488 1479 cursor=None,
1489 1480 warnings=True,
1490 1481 server=None,
1491 1482 verbose=True,
1492 1483 format=None,
1493 1484 oneDDict=None,
1494 1485 twoDDict=None,
1495 1486 ind2DList=None, **kwargs):
1496 1487
1497 1488 if not(self.isConfig):
1498 1489 self.setup(path=path,
1499 1490 startDate=startDate,
1500 1491 endDate=endDate,
1501 1492 startTime=startTime,
1502 1493 endTime=endTime,
1503 1494 set=set,
1504 1495 expLabel=expLabel,
1505 1496 ext=ext,
1506 1497 online=online,
1507 1498 delay=delay,
1508 1499 walk=walk,
1509 1500 getblock=getblock,
1510 1501 nTxs=nTxs,
1511 1502 realtime=realtime,
1512 1503 blocksize=blocksize,
1513 1504 blocktime=blocktime,
1514 1505 skip=skip,
1515 1506 cursor=cursor,
1516 1507 warnings=warnings,
1517 1508 server=server,
1518 1509 verbose=verbose,
1519 1510 format=format,
1520 1511 oneDDict=oneDDict,
1521 1512 twoDDict=twoDDict,
1522 1513 ind2DList=ind2DList)
1523 1514 self.isConfig = True
1524 1515 if server is None:
1525 1516 self.getData()
1526 1517 else:
1527 1518 self.getFromServer()
1528 1519
1529 1520
1530 1521 class JRODataWriter(JRODataIO):
1531 1522
1532 1523 """
1533 1524 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1534 1525 de los datos siempre se realiza por bloques.
1535 1526 """
1536 1527
1537 1528 blockIndex = 0
1538 1529
1539 1530 path = None
1540 1531
1541 1532 setFile = None
1542 1533
1543 1534 profilesPerBlock = None
1544 1535
1545 1536 blocksPerFile = None
1546 1537
1547 1538 nWriteBlocks = 0
1548 1539
1549 1540 fileDate = None
1550 1541
1551 1542 def __init__(self, dataOut=None):
1552 1543 raise NotImplementedError
1553 1544
1554 1545 def hasAllDataInBuffer(self):
1555 1546 raise NotImplementedError
1556 1547
1557 1548 def setBlockDimension(self):
1558 1549 raise NotImplementedError
1559 1550
1560 1551 def writeBlock(self):
1561 1552 raise NotImplementedError
1562 1553
1563 1554 def putData(self):
1564 1555 raise NotImplementedError
1565 1556
1566 1557 def getProcessFlags(self):
1567 1558
1568 1559 processFlags = 0
1569 1560
1570 1561 dtype_index = get_dtype_index(self.dtype)
1571 1562 procflag_dtype = get_procflag_dtype(dtype_index)
1572 1563
1573 1564 processFlags += procflag_dtype
1574 1565
1575 1566 if self.dataOut.flagDecodeData:
1576 1567 processFlags += PROCFLAG.DECODE_DATA
1577 1568
1578 1569 if self.dataOut.flagDeflipData:
1579 1570 processFlags += PROCFLAG.DEFLIP_DATA
1580 1571
1581 1572 if self.dataOut.code is not None:
1582 1573 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1583 1574
1584 1575 if self.dataOut.nCohInt > 1:
1585 1576 processFlags += PROCFLAG.COHERENT_INTEGRATION
1586 1577
1587 1578 if self.dataOut.type == "Spectra":
1588 1579 if self.dataOut.nIncohInt > 1:
1589 1580 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1590 1581
1591 1582 if self.dataOut.data_dc is not None:
1592 1583 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1593 1584
1594 1585 if self.dataOut.flagShiftFFT:
1595 1586 processFlags += PROCFLAG.SHIFT_FFT_DATA
1596 1587
1597 1588 return processFlags
1598 1589
1599 1590 def setBasicHeader(self):
1600 1591
1601 1592 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1602 1593 self.basicHeaderObj.version = self.versionFile
1603 1594 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1604 1595
1605 1596 utc = numpy.floor(self.dataOut.utctime)
1606 1597 milisecond = (self.dataOut.utctime - utc) * 1000.0
1607 1598
1608 1599 self.basicHeaderObj.utc = utc
1609 1600 self.basicHeaderObj.miliSecond = milisecond
1610 1601 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1611 1602 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1612 1603 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1613 1604
1614 1605 def setFirstHeader(self):
1615 1606 """
1616 1607 Obtiene una copia del First Header
1617 1608
1618 1609 Affected:
1619 1610
1620 1611 self.basicHeaderObj
1621 1612 self.systemHeaderObj
1622 1613 self.radarControllerHeaderObj
1623 1614 self.processingHeaderObj self.
1624 1615
1625 1616 Return:
1626 1617 None
1627 1618 """
1628 1619
1629 1620 raise NotImplementedError
1630 1621
1631 1622 def __writeFirstHeader(self):
1632 1623 """
1633 1624 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1634 1625
1635 1626 Affected:
1636 1627 __dataType
1637 1628
1638 1629 Return:
1639 1630 None
1640 1631 """
1641 1632
1642 1633 # CALCULAR PARAMETROS
1643 1634
1644 1635 sizeLongHeader = self.systemHeaderObj.size + \
1645 1636 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1646 1637 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1647 1638
1648 1639 self.basicHeaderObj.write(self.fp)
1649 1640 self.systemHeaderObj.write(self.fp)
1650 1641 self.radarControllerHeaderObj.write(self.fp)
1651 1642 self.processingHeaderObj.write(self.fp)
1652 1643
1653 1644 def __setNewBlock(self):
1654 1645 """
1655 1646 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1656 1647
1657 1648 Return:
1658 1649 0 : si no pudo escribir nada
1659 1650 1 : Si escribio el Basic el First Header
1660 1651 """
1661 1652 if self.fp == None:
1662 1653 self.setNextFile()
1663 1654
1664 1655 if self.flagIsNewFile:
1665 1656 return 1
1666 1657
1667 1658 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1668 1659 self.basicHeaderObj.write(self.fp)
1669 1660 return 1
1670 1661
1671 1662 if not(self.setNextFile()):
1672 1663 return 0
1673 1664
1674 1665 return 1
1675 1666
1676 1667 def writeNextBlock(self):
1677 1668 """
1678 1669 Selecciona el bloque siguiente de datos y los escribe en un file
1679 1670
1680 1671 Return:
1681 1672 0 : Si no hizo pudo escribir el bloque de datos
1682 1673 1 : Si no pudo escribir el bloque de datos
1683 1674 """
1684 1675 if not(self.__setNewBlock()):
1685 1676 return 0
1686 1677
1687 1678 self.writeBlock()
1688 1679
1689 1680 print "[Writing] Block No. %d/%d" % (self.blockIndex,
1690 1681 self.processingHeaderObj.dataBlocksPerFile)
1691 1682
1692 1683 return 1
1693 1684
1694 1685 def setNextFile(self):
1695 1686 """
1696 1687 Determina el siguiente file que sera escrito
1697 1688
1698 1689 Affected:
1699 1690 self.filename
1700 1691 self.subfolder
1701 1692 self.fp
1702 1693 self.setFile
1703 1694 self.flagIsNewFile
1704 1695
1705 1696 Return:
1706 1697 0 : Si el archivo no puede ser escrito
1707 1698 1 : Si el archivo esta listo para ser escrito
1708 1699 """
1709 1700 ext = self.ext
1710 1701 path = self.path
1711 1702
1712 1703 if self.fp != None:
1713 1704 self.fp.close()
1714 1705
1715 1706 timeTuple = time.localtime(self.dataOut.utctime)
1716 1707 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1717 1708
1718 1709 fullpath = os.path.join(path, subfolder)
1719 1710 setFile = self.setFile
1720 1711
1721 1712 if not(os.path.exists(fullpath)):
1722 1713 os.mkdir(fullpath)
1723 1714 setFile = -1 # inicializo mi contador de seteo
1724 1715 else:
1725 1716 filesList = os.listdir(fullpath)
1726 1717 if len(filesList) > 0:
1727 1718 filesList = sorted(filesList, key=str.lower)
1728 1719 filen = filesList[-1]
1729 1720 # el filename debera tener el siguiente formato
1730 1721 # 0 1234 567 89A BCDE (hex)
1731 1722 # x YYYY DDD SSS .ext
1732 1723 if isNumber(filen[8:11]):
1733 1724 # inicializo mi contador de seteo al seteo del ultimo file
1734 1725 setFile = int(filen[8:11])
1735 1726 else:
1736 1727 setFile = -1
1737 1728 else:
1738 1729 setFile = -1 # inicializo mi contador de seteo
1739 1730
1740 1731 setFile += 1
1741 1732
1742 1733 # If this is a new day it resets some values
1743 1734 if self.dataOut.datatime.date() > self.fileDate:
1744 1735 setFile = 0
1745 1736 self.nTotalBlocks = 0
1746
1747 filen = '%s%4.4d%3.3d%3.3d%s' % (
1748 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1737
1738 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1739 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1749 1740
1750 1741 filename = os.path.join(path, subfolder, filen)
1751 1742
1752 1743 fp = open(filename, 'wb')
1753 1744
1754 1745 self.blockIndex = 0
1755 1746
1756 1747 # guardando atributos
1757 1748 self.filename = filename
1758 1749 self.subfolder = subfolder
1759 1750 self.fp = fp
1760 1751 self.setFile = setFile
1761 1752 self.flagIsNewFile = 1
1762 1753 self.fileDate = self.dataOut.datatime.date()
1763 1754
1764 1755 self.setFirstHeader()
1765 1756
1766 1757 print '[Writing] Opening file: %s' % self.filename
1767 1758
1768 1759 self.__writeFirstHeader()
1769 1760
1770 1761 return 1
1771 1762
1772 1763 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1773 1764 """
1774 1765 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1775 1766
1776 1767 Inputs:
1777 1768 path : directory where data will be saved
1778 1769 profilesPerBlock : number of profiles per block
1779 1770 set : initial file set
1780 1771 datatype : An integer number that defines data type:
1781 1772 0 : int8 (1 byte)
1782 1773 1 : int16 (2 bytes)
1783 1774 2 : int32 (4 bytes)
1784 1775 3 : int64 (8 bytes)
1785 1776 4 : float32 (4 bytes)
1786 1777 5 : double64 (8 bytes)
1787 1778
1788 1779 Return:
1789 1780 0 : Si no realizo un buen seteo
1790 1781 1 : Si realizo un buen seteo
1791 1782 """
1792 1783
1793 1784 if ext == None:
1794 1785 ext = self.ext
1795 1786
1796 1787 self.ext = ext.lower()
1797 1788
1798 1789 self.path = path
1799
1790
1800 1791 if set is None:
1801 1792 self.setFile = -1
1802 1793 else:
1803 self.setFile = set - 1
1794 self.setFile = set - 1
1804 1795
1805 1796 self.blocksPerFile = blocksPerFile
1806 1797
1807 1798 self.profilesPerBlock = profilesPerBlock
1808 1799
1809 1800 self.dataOut = dataOut
1810 1801 self.fileDate = self.dataOut.datatime.date()
1811 1802 # By default
1812 1803 self.dtype = self.dataOut.dtype
1813 1804
1814 1805 if datatype is not None:
1815 1806 self.dtype = get_numpy_dtype(datatype)
1816 1807
1817 1808 if not(self.setNextFile()):
1818 1809 print "[Writing] There isn't a next file"
1819 1810 return 0
1820 1811
1821 1812 self.setBlockDimension()
1822 1813
1823 1814 return 1
1824 1815
1825 1816 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1826 1817
1827 1818 if not(self.isConfig):
1828 1819
1829 1820 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1830 1821 set=set, ext=ext, datatype=datatype, **kwargs)
1831 1822 self.isConfig = True
1832 1823
1833 1824 self.putData()
Show file before | Show file after | Hide comments
@@ -1,635 +1,631
1 1 '''
2 2 @author: Juan C. Espinoza
3 3 '''
4 4
5 5 import time
6 6 import json
7 7 import numpy
8 8 import paho.mqtt.client as mqtt
9 9 import zmq
10 10 import datetime
11 11 from zmq.utils.monitor import recv_monitor_message
12 12 from functools import wraps
13 13 from threading import Thread
14 14 from multiprocessing import Process
15 15
16 16 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
17 17 from schainpy.model.data.jrodata import JROData
18 18 from schainpy.utils import log
19 19
20 20 MAXNUMX = 100
21 21 MAXNUMY = 100
22 22
23 23 class PrettyFloat(float):
24 24 def __repr__(self):
25 25 return '%.2f' % self
26 26
27 27 def roundFloats(obj):
28 28 if isinstance(obj, list):
29 29 return map(roundFloats, obj)
30 30 elif isinstance(obj, float):
31 31 return round(obj, 2)
32 32
33 33 def decimate(z, MAXNUMY):
34 34 dy = int(len(z[0])/MAXNUMY) + 1
35 35
36 36 return z[::, ::dy]
37 37
38 38 class throttle(object):
39 39 '''
40 40 Decorator that prevents a function from being called more than once every
41 41 time period.
42 42 To create a function that cannot be called more than once a minute, but
43 43 will sleep until it can be called:
44 44 @throttle(minutes=1)
45 45 def foo():
46 46 pass
47 47
48 48 for i in range(10):
49 49 foo()
50 50 print "This function has run %s times." % i
51 51 '''
52 52
53 53 def __init__(self, seconds=0, minutes=0, hours=0):
54 54 self.throttle_period = datetime.timedelta(
55 55 seconds=seconds, minutes=minutes, hours=hours
56 56 )
57 57
58 58 self.time_of_last_call = datetime.datetime.min
59 59
60 60 def __call__(self, fn):
61 61 @wraps(fn)
62 62 def wrapper(*args, **kwargs):
63 63 coerce = kwargs.pop('coerce', None)
64 64 if coerce:
65 65 self.time_of_last_call = datetime.datetime.now()
66 66 return fn(*args, **kwargs)
67 67 else:
68 68 now = datetime.datetime.now()
69 69 time_since_last_call = now - self.time_of_last_call
70 70 time_left = self.throttle_period - time_since_last_call
71 71
72 72 if time_left > datetime.timedelta(seconds=0):
73 73 return
74 74
75 75 self.time_of_last_call = datetime.datetime.now()
76 76 return fn(*args, **kwargs)
77 77
78 78 return wrapper
79 79
80 80 class Data(object):
81 81 '''
82 82 Object to hold data to be plotted
83 83 '''
84 84
85 def __init__(self, plottypes, throttle_value):
85 def __init__(self, plottypes, throttle_value, exp_code):
86 86 self.plottypes = plottypes
87 87 self.throttle = throttle_value
88 self.exp_code = exp_code
88 89 self.ended = False
89 90 self.localtime = False
90 91 self.__times = []
91 92 self.__heights = []
92 93
93 94 def __str__(self):
94 95 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
95 96 return 'Data[{}][{}]'.format(';'.join(dum), len(self.__times))
96 97
97 98 def __len__(self):
98 99 return len(self.__times)
99 100
100 101 def __getitem__(self, key):
101 102 if key not in self.data:
102 103 raise KeyError(log.error('Missing key: {}'.format(key)))
103 104
104 105 if 'spc' in key:
105 106 ret = self.data[key]
106 107 else:
107 108 ret = numpy.array([self.data[key][x] for x in self.times])
108 109 if ret.ndim > 1:
109 110 ret = numpy.swapaxes(ret, 0, 1)
110 111 return ret
111 112
112 113 def __contains__(self, key):
113 114 return key in self.data
114 115
115 116 def setup(self):
116 117 '''
117 118 Configure object
118 119 '''
119 120
120 121 self.ended = False
121 122 self.data = {}
122 123 self.__times = []
123 124 self.__heights = []
124 125 self.__all_heights = set()
125 126 for plot in self.plottypes:
126 127 if 'snr' in plot:
127 128 plot = 'snr'
128 129 self.data[plot] = {}
129 130
130 131 def shape(self, key):
131 132 '''
132 133 Get the shape of the one-element data for the given key
133 134 '''
134 135
135 136 if len(self.data[key]):
136 137 if 'spc' in key:
137 138 return self.data[key].shape
138 139 return self.data[key][self.__times[0]].shape
139 140 return (0,)
140 141
141 142 def update(self, dataOut, tm):
142 143 '''
143 144 Update data object with new dataOut
144 145 '''
145 146
146 147 if tm in self.__times:
147 148 return
148 149
149 150 self.parameters = getattr(dataOut, 'parameters', [])
150 self.pairs = dataOut.pairsList
151 if hasattr(dataOut, 'pairsList'):
152 self.pairs = dataOut.pairsList
151 153 self.channels = dataOut.channelList
152 154 self.interval = dataOut.getTimeInterval()
153 155 self.localtime = dataOut.useLocalTime
154 156 if 'spc' in self.plottypes or 'cspc' in self.plottypes:
155 157 self.xrange = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
156 158 self.__heights.append(dataOut.heightList)
157 159 self.__all_heights.update(dataOut.heightList)
158 160 self.__times.append(tm)
159 161
160 162 for plot in self.plottypes:
161 163 if plot == 'spc':
162 164 z = dataOut.data_spc/dataOut.normFactor
163 165 self.data[plot] = 10*numpy.log10(z)
164 166 if plot == 'cspc':
165 167 self.data[plot] = dataOut.data_cspc
166 168 if plot == 'noise':
167 169 self.data[plot][tm] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
168 170 if plot == 'rti':
169 171 self.data[plot][tm] = dataOut.getPower()
170 172 if plot == 'snr_db':
171 173 self.data['snr'][tm] = dataOut.data_SNR
172 174 if plot == 'snr':
173 175 self.data[plot][tm] = 10*numpy.log10(dataOut.data_SNR)
174 176 if plot == 'dop':
175 177 self.data[plot][tm] = 10*numpy.log10(dataOut.data_DOP)
176 178 if plot == 'mean':
177 179 self.data[plot][tm] = dataOut.data_MEAN
178 180 if plot == 'std':
179 181 self.data[plot][tm] = dataOut.data_STD
180 182 if plot == 'coh':
181 183 self.data[plot][tm] = dataOut.getCoherence()
182 184 if plot == 'phase':
183 185 self.data[plot][tm] = dataOut.getCoherence(phase=True)
184 186 if plot == 'output':
185 187 self.data[plot][tm] = dataOut.data_output
186 188 if plot == 'param':
187 189 self.data[plot][tm] = dataOut.data_param
188 190
189 191 def normalize_heights(self):
190 192 '''
191 193 Ensure same-dimension of the data for different heighList
192 194 '''
193 195
194 196 H = numpy.array(list(self.__all_heights))
195 197 H.sort()
196 198 for key in self.data:
197 199 shape = self.shape(key)[:-1] + H.shape
198 200 for tm, obj in self.data[key].items():
199 201 h = self.__heights[self.__times.index(tm)]
200 202 if H.size == h.size:
201 203 continue
202 204 index = numpy.where(numpy.in1d(H, h))[0]
203 205 dummy = numpy.zeros(shape) + numpy.nan
204 206 if len(shape) == 2:
205 207 dummy[:, index] = obj
206 208 else:
207 209 dummy[index] = obj
208 210 self.data[key][tm] = dummy
209 211
210 212 self.__heights = [H for tm in self.__times]
211 213
212 214 def jsonify(self, decimate=False):
213 215 '''
214 216 Convert data to json
215 217 '''
216 218
217 ret = {}
219 data = {}
218 220 tm = self.times[-1]
219
220 for key, value in self.data:
221
222 for key in self.data:
221 223 if key in ('spc', 'cspc'):
222 ret[key] = roundFloats(self.data[key].to_list())
224 dx = int(self.data[key].shape[1]/MAXNUMX) + 1
225 dy = int(self.data[key].shape[2]/MAXNUMY) + 1
226 data[key] = roundFloats(self.data[key][::, ::dx, ::dy].tolist())
223 227 else:
224 ret[key] = roundFloats(self.data[key][tm].to_list())
228 data[key] = roundFloats(self.data[key][tm].tolist())
225 229
226 ret['timestamp'] = tm
230 ret = {'data': data}
231 ret['exp_code'] = self.exp_code
232 ret['time'] = tm
227 233 ret['interval'] = self.interval
234 ret['localtime'] = self.localtime
235 ret['yrange'] = roundFloats(self.heights.tolist())
236 if key in ('spc', 'cspc'):
237 ret['xrange'] = roundFloats(self.xrange[2][::dx].tolist())
238 if hasattr(self, 'pairs'):
239 ret['pairs'] = self.pairs
240 return json.dumps(ret)
228 241
229 242 @property
230 243 def times(self):
231 244 '''
232 245 Return the list of times of the current data
233 246 '''
234 247
235 248 ret = numpy.array(self.__times)
236 249 ret.sort()
237 250 return ret
238 251
239 252 @property
240 253 def heights(self):
241 254 '''
242 255 Return the list of heights of the current data
243 256 '''
244 257
245 258 return numpy.array(self.__heights[-1])
246 259
247 260 class PublishData(Operation):
248 261 '''
249 262 Operation to send data over zmq.
250 263 '''
251 264
252 265 __attrs__ = ['host', 'port', 'delay', 'zeromq', 'mqtt', 'verbose']
253 266
254 267 def __init__(self, **kwargs):
255 268 """Inicio."""
256 269 Operation.__init__(self, **kwargs)
257 270 self.isConfig = False
258 271 self.client = None
259 272 self.zeromq = None
260 273 self.mqtt = None
261 274
262 275 def on_disconnect(self, client, userdata, rc):
263 276 if rc != 0:
264 277 log.warning('Unexpected disconnection.')
265 278 self.connect()
266 279
267 280 def connect(self):
268 281 log.warning('trying to connect')
269 282 try:
270 283 self.client.connect(
271 284 host=self.host,
272 285 port=self.port,
273 286 keepalive=60*10,
274 287 bind_address='')
275 288 self.client.loop_start()
276 289 # self.client.publish(
277 290 # self.topic + 'SETUP',
278 291 # json.dumps(setup),
279 292 # retain=True
280 293 # )
281 294 except:
282 295 log.error('MQTT Conection error.')
283 296 self.client = False
284 297
285 298 def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, verbose=True, **kwargs):
286 299 self.counter = 0
287 300 self.topic = kwargs.get('topic', 'schain')
288 301 self.delay = kwargs.get('delay', 0)
289 302 self.plottype = kwargs.get('plottype', 'spectra')
290 303 self.host = kwargs.get('host', "10.10.10.82")
291 304 self.port = kwargs.get('port', 3000)
292 305 self.clientId = clientId
293 306 self.cnt = 0
294 307 self.zeromq = zeromq
295 308 self.mqtt = kwargs.get('plottype', 0)
296 309 self.client = None
297 310 self.verbose = verbose
298 311 setup = []
299 312 if mqtt is 1:
300 313 self.client = mqtt.Client(
301 314 client_id=self.clientId + self.topic + 'SCHAIN',
302 315 clean_session=True)
303 316 self.client.on_disconnect = self.on_disconnect
304 317 self.connect()
305 318 for plot in self.plottype:
306 319 setup.append({
307 320 'plot': plot,
308 321 'topic': self.topic + plot,
309 322 'title': getattr(self, plot + '_' + 'title', False),
310 323 'xlabel': getattr(self, plot + '_' + 'xlabel', False),
311 324 'ylabel': getattr(self, plot + '_' + 'ylabel', False),
312 325 'xrange': getattr(self, plot + '_' + 'xrange', False),
313 326 'yrange': getattr(self, plot + '_' + 'yrange', False),
314 327 'zrange': getattr(self, plot + '_' + 'zrange', False),
315 328 })
316 329 if zeromq is 1:
317 330 context = zmq.Context()
318 331 self.zmq_socket = context.socket(zmq.PUSH)
319 332 server = kwargs.get('server', 'zmq.pipe')
320 333
321 334 if 'tcp://' in server:
322 335 address = server
323 336 else:
324 337 address = 'ipc:///tmp/%s' % server
325 338
326 339 self.zmq_socket.connect(address)
327 340 time.sleep(1)
328 341
329 342
330 343 def publish_data(self):
331 344 self.dataOut.finished = False
332 345 if self.mqtt is 1:
333 346 yData = self.dataOut.heightList[:2].tolist()
334 347 if self.plottype == 'spectra':
335 348 data = getattr(self.dataOut, 'data_spc')
336 349 z = data/self.dataOut.normFactor
337 350 zdB = 10*numpy.log10(z)
338 351 xlen, ylen = zdB[0].shape
339 352 dx = int(xlen/MAXNUMX) + 1
340 353 dy = int(ylen/MAXNUMY) + 1
341 354 Z = [0 for i in self.dataOut.channelList]
342 355 for i in self.dataOut.channelList:
343 356 Z[i] = zdB[i][::dx, ::dy].tolist()
344 357 payload = {
345 358 'timestamp': self.dataOut.utctime,
346 359 'data': roundFloats(Z),
347 360 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
348 361 'interval': self.dataOut.getTimeInterval(),
349 362 'type': self.plottype,
350 363 'yData': yData
351 364 }
352 365
353 366 elif self.plottype in ('rti', 'power'):
354 367 data = getattr(self.dataOut, 'data_spc')
355 368 z = data/self.dataOut.normFactor
356 369 avg = numpy.average(z, axis=1)
357 370 avgdB = 10*numpy.log10(avg)
358 371 xlen, ylen = z[0].shape
359 372 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
360 373 AVG = [0 for i in self.dataOut.channelList]
361 374 for i in self.dataOut.channelList:
362 375 AVG[i] = avgdB[i][::dy].tolist()
363 376 payload = {
364 377 'timestamp': self.dataOut.utctime,
365 378 'data': roundFloats(AVG),
366 379 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
367 380 'interval': self.dataOut.getTimeInterval(),
368 381 'type': self.plottype,
369 382 'yData': yData
370 383 }
371 384 elif self.plottype == 'noise':
372 385 noise = self.dataOut.getNoise()/self.dataOut.normFactor
373 386 noisedB = 10*numpy.log10(noise)
374 387 payload = {
375 388 'timestamp': self.dataOut.utctime,
376 389 'data': roundFloats(noisedB.reshape(-1, 1).tolist()),
377 390 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
378 391 'interval': self.dataOut.getTimeInterval(),
379 392 'type': self.plottype,
380 393 'yData': yData
381 394 }
382 395 elif self.plottype == 'snr':
383 396 data = getattr(self.dataOut, 'data_SNR')
384 397 avgdB = 10*numpy.log10(data)
385 398
386 399 ylen = data[0].size
387 400 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
388 401 AVG = [0 for i in self.dataOut.channelList]
389 402 for i in self.dataOut.channelList:
390 403 AVG[i] = avgdB[i][::dy].tolist()
391 404 payload = {
392 405 'timestamp': self.dataOut.utctime,
393 406 'data': roundFloats(AVG),
394 407 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
395 408 'type': self.plottype,
396 409 'yData': yData
397 410 }
398 411 else:
399 412 print "Tipo de grafico invalido"
400 413 payload = {
401 414 'data': 'None',
402 415 'timestamp': 'None',
403 416 'type': None
404 417 }
405 418
406 419 self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
407 420
408 421 if self.zeromq is 1:
409 422 if self.verbose:
410 423 log.log(
411 424 'Sending {} - {}'.format(self.dataOut.type, self.dataOut.datatime),
412 425 self.name
413 426 )
414 427 self.zmq_socket.send_pyobj(self.dataOut)
415 428
416 429 def run(self, dataOut, **kwargs):
417 430 self.dataOut = dataOut
418 431 if not self.isConfig:
419 432 self.setup(**kwargs)
420 433 self.isConfig = True
421 434
422 435 self.publish_data()
423 436 time.sleep(self.delay)
424 437
425 438 def close(self):
426 439 if self.zeromq is 1:
427 440 self.dataOut.finished = True
428 441 self.zmq_socket.send_pyobj(self.dataOut)
429 442 time.sleep(0.1)
430 443 self.zmq_socket.close()
431 444 if self.client:
432 445 self.client.loop_stop()
433 446 self.client.disconnect()
434 447
435 448
436 449 class ReceiverData(ProcessingUnit):
437 450
438 451 __attrs__ = ['server']
439 452
440 453 def __init__(self, **kwargs):
441 454
442 455 ProcessingUnit.__init__(self, **kwargs)
443 456
444 457 self.isConfig = False
445 458 server = kwargs.get('server', 'zmq.pipe')
446 459 if 'tcp://' in server:
447 460 address = server
448 461 else:
449 462 address = 'ipc:///tmp/%s' % server
450 463
451 464 self.address = address
452 465 self.dataOut = JROData()
453 466
454 467 def setup(self):
455 468
456 469 self.context = zmq.Context()
457 470 self.receiver = self.context.socket(zmq.PULL)
458 471 self.receiver.bind(self.address)
459 472 time.sleep(0.5)
460 473 log.success('ReceiverData from {}'.format(self.address))
461 474
462 475
463 476 def run(self):
464 477
465 478 if not self.isConfig:
466 479 self.setup()
467 480 self.isConfig = True
468 481
469 482 self.dataOut = self.receiver.recv_pyobj()
470 483 log.log('{} - {}'.format(self.dataOut.type,
471 484 self.dataOut.datatime.ctime(),),
472 485 'Receiving')
473 486
474 487
475 488 class PlotterReceiver(ProcessingUnit, Process):
476 489
477 490 throttle_value = 5
478 __attrs__ = ['server', 'plottypes', 'realtime', 'localtime', 'throttle']
491 __attrs__ = ['server', 'plottypes', 'realtime', 'localtime', 'throttle',
492 'exp_code', 'web_server']
479 493
480 494 def __init__(self, **kwargs):
481 495
482 496 ProcessingUnit.__init__(self, **kwargs)
483 497 Process.__init__(self)
484 498 self.mp = False
485 499 self.isConfig = False
486 500 self.isWebConfig = False
487 501 self.connections = 0
488 502 server = kwargs.get('server', 'zmq.pipe')
489 plot_server = kwargs.get('plot_server', 'zmq.web')
503 web_server = kwargs.get('web_server', None)
490 504 if 'tcp://' in server:
491 505 address = server
492 506 else:
493 507 address = 'ipc:///tmp/%s' % server
494
495 if 'tcp://' in plot_server:
496 plot_address = plot_server
497 else:
498 plot_address = 'ipc:///tmp/%s' % plot_server
499
500 508 self.address = address
501 self.plot_address = plot_address
509 self.web_address = web_server
502 510 self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
503 511 self.realtime = kwargs.get('realtime', False)
504 512 self.localtime = kwargs.get('localtime', True)
505 513 self.throttle_value = kwargs.get('throttle', 5)
514 self.exp_code = kwargs.get('exp_code', None)
506 515 self.sendData = self.initThrottle(self.throttle_value)
507 516 self.dates = []
508 517 self.setup()
509 518
510 519 def setup(self):
511 520
512 self.data = Data(self.plottypes, self.throttle_value)
521 self.data = Data(self.plottypes, self.throttle_value, self.exp_code)
513 522 self.isConfig = True
514 523
515 524 def event_monitor(self, monitor):
516 525
517 526 events = {}
518 527
519 528 for name in dir(zmq):
520 529 if name.startswith('EVENT_'):
521 530 value = getattr(zmq, name)
522 531 events[value] = name
523 532
524 533 while monitor.poll():
525 534 evt = recv_monitor_message(monitor)
526 535 if evt['event'] == 32:
527 536 self.connections += 1
528 537 if evt['event'] == 512:
529 538 pass
530 539
531 540 evt.update({'description': events[evt['event']]})
532 541
533 542 if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
534 543 break
535 544 monitor.close()
536 545 print('event monitor thread done!')
537 546
538 547 def initThrottle(self, throttle_value):
539 548
540 549 @throttle(seconds=throttle_value)
541 550 def sendDataThrottled(fn_sender, data):
542 551 fn_sender(data)
543 552
544 553 return sendDataThrottled
545 554
546 555 def send(self, data):
547 556 log.success('Sending {}'.format(data), self.name)
548 557 self.sender.send_pyobj(data)
549 558
550 559 def run(self):
551 560
552 561 log.success(
553 562 'Starting from {}'.format(self.address),
554 563 self.name
555 564 )
556 565
557 566 self.context = zmq.Context()
558 567 self.receiver = self.context.socket(zmq.PULL)
559 568 self.receiver.bind(self.address)
560 569 monitor = self.receiver.get_monitor_socket()
561 570 self.sender = self.context.socket(zmq.PUB)
562 if self.realtime:
571 if self.web_address:
572 log.success(
573 'Sending to web: {}'.format(self.web_address),
574 self.name
575 )
563 576 self.sender_web = self.context.socket(zmq.PUB)
564 self.sender_web.connect(self.plot_address)
577 self.sender_web.connect(self.web_address)
565 578 time.sleep(1)
566 579
567 580 if 'server' in self.kwargs:
568 581 self.sender.bind("ipc:///tmp/{}.plots".format(self.kwargs['server']))
569 582 else:
570 583 self.sender.bind("ipc:///tmp/zmq.plots")
571 584
572 585 time.sleep(2)
573 586
574 587 t = Thread(target=self.event_monitor, args=(monitor,))
575 588 t.start()
576 589
577 590 while True:
578 591 dataOut = self.receiver.recv_pyobj()
579 592 if not dataOut.flagNoData:
580 593 if dataOut.type == 'Parameters':
581 594 tm = dataOut.utctimeInit
582 595 else:
583 596 tm = dataOut.utctime
584 597 if dataOut.useLocalTime:
585 598 if not self.localtime:
586 599 tm += time.timezone
587 600 dt = datetime.datetime.fromtimestamp(tm).date()
588 601 else:
589 602 if self.localtime:
590 603 tm -= time.timezone
591 604 dt = datetime.datetime.utcfromtimestamp(tm).date()
592 605 coerce = False
593 606 if dt not in self.dates:
594 607 if self.data:
595 608 self.data.ended = True
596 609 self.send(self.data)
597 610 coerce = True
598 611 self.data.setup()
599 612 self.dates.append(dt)
600 613
601 614 self.data.update(dataOut, tm)
602 615
603 616 if dataOut.finished is True:
604 617 self.connections -= 1
605 618 if self.connections == 0 and dt in self.dates:
606 619 self.data.ended = True
607 620 self.send(self.data)
608 621 self.data.setup()
609 622 else:
610 623 if self.realtime:
611 624 self.send(self.data)
612 # self.sender_web.send_string(self.data.jsonify())
625 if self.web_address:
626 self.sender_web.send(self.data.jsonify())
613 627 else:
614 628 self.sendData(self.send, self.data, coerce=coerce)
615 629 coerce = False
616 630
617 631 return
618
619 def sendToWeb(self):
620
621 if not self.isWebConfig:
622 context = zmq.Context()
623 sender_web_config = context.socket(zmq.PUB)
624 if 'tcp://' in self.plot_address:
625 dum, address, port = self.plot_address.split(':')
626 conf_address = '{}:{}:{}'.format(dum, address, int(port)+1)
627 else:
628 conf_address = self.plot_address + '.config'
629 sender_web_config.bind(conf_address)
630 time.sleep(1)
631 for kwargs in self.operationKwargs.values():
632 if 'plot' in kwargs:
633 log.success('[Sending] Config data to web for {}'.format(kwargs['code'].upper()))
634 sender_web_config.send_string(json.dumps(kwargs))
635 self.isWebConfig = True
Show file before | Show file after | Hide comments
@@ -1,79 +1,69
1 1 '''
2 2 Created on Jul 16, 2014
3 3
4 4 @author: Miguel Urco
5 5 '''
6 6
7 7 import os
8 8 from setuptools import setup, Extension
9 9 from setuptools.command.build_ext import build_ext as _build_ext
10 10 from schainpy import __version__
11 from schainpy.utils import log
12
13 11
14 12 class build_ext(_build_ext):
15 13 def finalize_options(self):
16 14 _build_ext.finalize_options(self)
17 15 # Prevent numpy from thinking it is still in its setup process:
18 16 __builtins__.__NUMPY_SETUP__ = False
19 17 import numpy
20 18 self.include_dirs.append(numpy.get_include())
21 19
22
23 try:
24 from PyQt4 import QtCore, QtGui
25 from PyQt4.QtGui import QApplication
26 except:
27 log.warning(
28 'You should install PyQt4 module in order to run the GUI. See the README.')
29
30
31 setup(name="schainpy",
32 version=__version__,
33 description="Python tools to read, write and process Jicamarca data",
34 author="Miguel Urco",
35 author_email="miguel.urco@jro.igp.gob.pe",
36 url="http://jro.igp.gob.pe",
37 packages={'schainpy',
20 setup(name = "schainpy",
21 version = __version__,
22 description = "Python tools to read, write and process Jicamarca data",
23 author = "Miguel Urco",
24 author_email = "miguel.urco@jro.igp.gob.pe",
25 url = "http://jro.igp.gob.pe",
26 packages = {'schainpy',
38 27 'schainpy.model',
39 28 'schainpy.model.data',
40 29 'schainpy.model.graphics',
41 30 'schainpy.model.io',
42 31 'schainpy.model.proc',
43 32 'schainpy.model.serializer',
44 33 'schainpy.model.utils',
34 'schainpy.utils',
45 35 'schainpy.gui',
46 36 'schainpy.gui.figures',
47 37 'schainpy.gui.viewcontroller',
48 38 'schainpy.gui.viewer',
49 'schainpy.gui.viewer.windows'},
50 ext_package='schainpy',
51 py_modules=[''],
52 package_data={'': ['schain.conf.template'],
53 'schainpy.gui.figures': ['*.png', '*.jpg'],
54 },
55 include_package_data=False,
56 scripts=['schainpy/gui/schainGUI'],
57 ext_modules=[
58 Extension("cSchain", ["schainpy/model/proc/extensions.c"]
59 )],
60 entry_points={
39 'schainpy.gui.viewer.windows',
40 'schainpy.cli'},
41 ext_package = 'schainpy',
42 package_data = {'': ['schain.conf.template'],
43 'schainpy.gui.figures': ['*.png', '*.jpg'],
44 },
45 include_package_data = False,
46 scripts = ['schainpy/gui/schainGUI'],
47 ext_modules = [
48 Extension("cSchain", ["schainpy/model/proc/extensions.c"])
49 ],
50 entry_points = {
61 51 'console_scripts': [
62 'schain = schaincli.cli:main',
52 'schain = schainpy.cli.cli:main',
63 53 ],
64 54 },
65 cmdclass={'build_ext': build_ext},
66 setup_requires=["numpy >= 1.11.2"],
67 install_requires=[
55 cmdclass = {'build_ext': build_ext},
56 setup_requires = ["numpy >= 1.11.2"],
57 install_requires = [
68 58 "scipy >= 0.14.0",
69 59 "h5py >= 2.2.1",
70 "matplotlib >= 1.4.2",
60 "matplotlib >= 2.0.0",
71 61 "pyfits >= 3.4",
72 62 "paramiko >= 2.1.2",
73 63 "paho-mqtt >= 1.2",
74 64 "zmq",
75 65 "fuzzywuzzy",
76 66 "click",
77 67 "python-Levenshtein"
78 ],
79 )
68 ],
69 )
General Comments 0
You need to be logged in to leave comments. Login now