| @@ -0,0 +1,26 | |||
|
|
1 | FROM python:2.7-slim | |
|
|
2 | ||
|
|
3 | RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommends \ | |
|
|
4 | git \ | |
|
|
5 | gcc \ | |
|
|
6 | libpng-dev \ | |
|
|
7 | libfreetype6-dev \ | |
|
|
8 | libopenblas-dev \ | |
|
|
9 | liblapack-dev \ | |
|
|
10 | libatlas-base-dev \ | |
|
|
11 | libssl-dev \ | |
|
|
12 | libhdf5-dev \ | |
|
|
13 | && git clone --branch v2.3 --depth 1 \ | |
|
|
14 | http://jro-dev.igp.gob.pe/rhodecode/schain \ | |
|
|
15 | && pip install numpy \ | |
|
|
16 | && cd schain \ | |
|
|
17 | && pip install . \ | |
|
|
18 | && rm -rf * \ | |
|
|
19 | && apt-get purge -y --auto-remove git gcc \ | |
|
|
20 | && rm -rf /var/lib/apt/lists/* | |
|
|
21 | ||
|
|
22 | ENV BACKEND="Agg" | |
|
|
23 | ||
|
|
24 | VOLUME /data | |
|
|
25 | ||
|
|
26 | ENTRYPOINT ["schain"] | |
| @@ -6,7 +6,7 Signal Chain (SCh) is a radar data processing library developed using [Python](w | |||
|
|
6 | 6 | |
|
|
7 | 7 | ## Installation |
|
|
8 | 8 | |
|
|
9 |
Install system dependencies, clone the latest version from [ |
|
|
|
9 | Install system dependencies, clone the latest version from [here](http://jro-dev.igp.gob.pe/rhodecode/schain/) and install it as a normal python package. | |
|
|
10 | 10 | |
|
|
11 | 11 | ### Linux based system |
|
|
12 | 12 | ``` |
| @@ -16,36 +16,38 $ cd schain | |||
|
|
16 | 16 | $ sudo pip install ./ |
|
|
17 | 17 | |
|
|
18 | 18 | ``` |
|
|
19 | **It is recommended to install schain in a virtual environment** | |
|
|
20 | ``` | |
|
|
21 | $ sudo pip install virtualenv | |
|
|
22 | $ virtualenv /path/to/virtual --system-site-packages | |
|
|
23 | $ source /path/to/virtual/bin/activate | |
|
|
24 | (virtual) $ cd schain | |
|
|
25 | (virtual) $ pip install ./ | |
|
|
26 | ||
|
|
27 | ``` | |
|
|
28 | 19 | |
|
|
29 | 20 | ### MAC Os |
|
|
30 | 21 | ``` |
|
|
22 | $ brew install python | |
|
|
31 | 23 |
|
|
|
32 | 24 | $ git clone http://jro-dev.igp.gob.pe/rhodecode/schain/ |
|
|
33 | 25 | $ cd schain |
|
|
34 | 26 | $ pip install ./ |
|
|
35 | 27 | ``` |
|
|
36 | 28 | |
|
|
37 | if ```pip install ./``` does not work, install a proper python enviroment, and repeat the steps. | |
|
|
29 | **It is recommended to install schain in a virtual environment** | |
|
|
38 | 30 | ``` |
|
|
39 | $ brew install python | |
|
|
31 | $ virtualenv /path/to/virtual | |
|
|
32 | $ source /path/to/virtual/bin/activate | |
|
|
33 | (virtual) $ cd schain | |
|
|
34 | (virtual) $ pip install ./ | |
|
|
35 | (virtual) $ bash link_PyQt4.sh | |
|
|
40 | 36 | ``` |
|
|
41 | 37 | |
|
|
42 | ### GUI Installation | |
|
|
38 | ### Docker | |
|
|
39 | ||
|
|
40 | Download Dockerfile from the repository, and create a docker image | |
|
|
43 | 41 | |
|
|
44 | 42 | ``` |
|
|
45 | $ sudo apt-get install python-pip python-dev gfortran libpng-dev freetype* libblas-dev liblapack-dev libatlas-base-dev python-qt4 python-tk libssl-dev libhdf5-dev | |
|
|
46 | $ (virtual) bash link_PyQt4.sh | |
|
|
43 | $ docker build -t schain . | |
|
|
47 | 44 | ``` |
|
|
48 | 45 | |
|
|
46 | You can run a container using an xml file or a schain script also you need to mount a volume for the data input and for the output files/plots | |
|
|
47 | ``` | |
|
|
48 | $ docker run -it --rm --volume /path/to/host/data:/data schain xml /data/test.xml | |
|
|
49 | $ docker run -it --rm --volume /path/to/host/data:/data --entrypoint=/bin/python schain /data/test.py | |
|
|
50 | ``` | |
|
|
49 | 51 | |
|
|
50 | 52 | ## First Script |
|
|
51 | 53 | |
| @@ -1,8 +1,5 | |||
|
|
1 | 1 | #!/bin/bash |
|
|
2 | 2 | # This hook is run after a new virtualenv is activated. |
|
|
3 | # ~/.virtualenvs/postmkvirtualenv | |
|
|
4 | ||
|
|
5 | libs=( PyQt4 sip.so ) | |
|
|
6 | 3 | |
|
|
7 | 4 | python_version=python$(python -c "import sys; print (str(sys.version_info[0])+'.'+str(sys.version_info[1]))") |
|
|
8 | 5 | var=( $(which -a $python_version) ) |
| @@ -10,8 +7,9 var=( $(which -a $python_version) ) | |||
|
|
10 | 7 | get_python_lib_cmd="from distutils.sysconfig import get_python_lib; print (get_python_lib())" |
|
|
11 | 8 | lib_virtualenv_path=$(python -c "$get_python_lib_cmd") |
|
|
12 | 9 | lib_system_path=$(${var[-1]} -c "$get_python_lib_cmd") |
|
|
10 | sip_path=$(ls $lib_system_path/sip*.so) | |
|
|
13 | 11 | |
|
|
14 | for lib in ${libs[@]} | |
|
|
15 | do | |
|
|
16 | ln -s $lib_system_path/$lib $lib_virtualenv_path/$lib | |
|
|
17 | done | |
|
|
12 | echo "Linking Qt4..." | |
|
|
13 | ln -s $lib_system_path/PyQt4 $lib_virtualenv_path/PyQt4 | |
|
|
14 | echo "Linking SIP..." | |
|
|
15 | ln -s $sip_path $lib_virtualenv_path/sip.so | |
|
|
1 | NO CONTENT: file renamed from schaincli/README.md to schainpy/cli/README.md |
|
|
1 | NO CONTENT: file renamed from schaincli/__init__.py to schainpy/cli/__init__.py |
| @@ -7,7 +7,6 import glob | |||
|
|
7 | 7 | save_stdout = sys.stdout |
|
|
8 | 8 | sys.stdout = open('trash', 'w') |
|
|
9 | 9 | from multiprocessing import cpu_count |
|
|
10 | from schaincli import templates | |
|
|
11 | 10 | from schainpy.controller import Project |
|
|
12 | 11 | from schainpy.model import Operation, ProcessingUnit |
|
|
13 | 12 | from schainpy.utils import log |
| @@ -15,6 +14,7 from importlib import import_module | |||
|
|
15 | 14 | from pydoc import locate |
|
|
16 | 15 | from fuzzywuzzy import process |
|
|
17 | 16 | from schainpy.utils import paramsFinder |
|
|
17 | import templates | |
|
|
18 | 18 | sys.stdout = save_stdout |
|
|
19 | 19 | |
|
|
20 | 20 | |
|
|
1 | NO CONTENT: file renamed from schaincli/templates.py to schainpy/cli/templates.py |
|
|
1 | NO CONTENT: file renamed from schaincli/tests/__init__.py to schainpy/cli/tests/__init__.py |
| @@ -1,6 +1,6 | |||
|
|
1 | 1 | import pytest |
|
|
2 | 2 | from click.testing import CliRunner |
|
|
3 | from schaincli import cli | |
|
|
3 | from schainpy.cli import cli | |
|
|
4 | 4 | |
|
|
5 | 5 | |
|
|
6 | 6 | @pytest.fixture |
| @@ -34,6 +34,8 def MPProject(project, n=cpu_count()): | |||
|
|
34 | 34 | op = rconf.getOperationObj('run') |
|
|
35 | 35 | dt1 = op.getParameterValue('startDate') |
|
|
36 | 36 | dt2 = op.getParameterValue('endDate') |
|
|
37 | tm1 = op.getParameterValue('startTime') | |
|
|
38 | tm2 = op.getParameterValue('endTime') | |
|
|
37 | 39 | days = (dt2 - dt1).days |
|
|
38 | 40 | |
|
|
39 | 41 | for day in range(days + 1): |
| @@ -46,11 +48,13 def MPProject(project, n=cpu_count()): | |||
|
|
46 | 48 | paths, files = reader.searchFilesOffLine(path=rconf.path, |
|
|
47 | 49 | startDate=dt, |
|
|
48 | 50 | endDate=dt, |
|
|
51 | startTime=tm1, | |
|
|
52 | endTime=tm2, | |
|
|
49 | 53 | ext=DTYPES[rconf.datatype]) |
|
|
50 | 54 | nFiles = len(files) |
|
|
51 | 55 | if nFiles == 0: |
|
|
52 | 56 | continue |
|
|
53 | skip = int(math.ceil(nFiles / n)) | |
|
|
57 | skip = int(math.ceil(nFiles / n)) | |
|
|
54 | 58 | while nFiles > cursor * skip: |
|
|
55 | 59 | rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor, |
|
|
56 | 60 | skip=skip) |
| @@ -962,7 +966,7 class Project(Process): | |||
|
|
962 | 966 | print '*' * 60 |
|
|
963 | 967 | |
|
|
964 | 968 | self.id = str(id) |
|
|
965 | self.description = description | |
|
|
969 | self.description = description | |
|
|
966 | 970 | |
|
|
967 | 971 | def update(self, name, description): |
|
|
968 | 972 | |
| @@ -1270,7 +1274,7 class Project(Process): | |||
|
|
1270 | 1274 | def run(self): |
|
|
1271 | 1275 | |
|
|
1272 | 1276 | log.success('Starting {}'.format(self.name)) |
|
|
1273 | ||
|
|
1277 | self.start_time = time.time() | |
|
|
1274 | 1278 | self.createObjects() |
|
|
1275 | 1279 | self.connectObjects() |
|
|
1276 | 1280 | |
| @@ -1314,4 +1318,6 class Project(Process): | |||
|
|
1314 | 1318 | procUnitConfObj = self.procUnitConfObjDict[procKey] |
|
|
1315 | 1319 | procUnitConfObj.close() |
|
|
1316 | 1320 | |
|
|
1317 |
log.success('{} finished'.format( |
|
|
|
1321 | log.success('{} finished (time: {}s)'.format( | |
|
|
1322 | self.name, | |
|
|
1323 | time.time()-self.start_time)) | |
| @@ -8,10 +8,9 try: | |||
|
|
8 | 8 | from PyQt4.QtGui import QApplication |
|
|
9 | 9 | except: |
|
|
10 | 10 | log.error( |
|
|
11 |
'You should install P |
|
|
|
11 | 'You should install PyQt4 module in order to run the GUI. See the README.') | |
|
|
12 | 12 | sys.exit() |
|
|
13 | 13 | |
|
|
14 | ||
|
|
15 | 14 | from schainpy.gui.viewcontroller.initwindow import InitWindow |
|
|
16 | 15 | from schainpy.gui.viewcontroller.basicwindow import BasicWindow |
|
|
17 | 16 | from schainpy.gui.viewcontroller.workspace import Workspace |
| @@ -507,7 +507,7 class RadarControllerHeader(Header): | |||
|
|
507 | 507 | code1 = (self.code + 1.0) / 2. |
|
|
508 | 508 | |
|
|
509 | 509 | for ic in range(self.nCode): |
|
|
510 | tempx = numpy.zeros(int(self.nBaud / 32.)) | |
|
|
510 | tempx = numpy.zeros(int(numpy.ceil(self.nBaud / 32.))) | |
|
|
511 | 511 | start = 0 |
|
|
512 | 512 | end = 32 |
|
|
513 | 513 | for i in range(len(tempx)): |
| @@ -22,7 +22,7 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list( | |||
|
|
22 | 22 | 'jro', numpy.vstack((blu_values, jet_values))) |
|
|
23 | 23 | matplotlib.pyplot.register_cmap(cmap=ncmap) |
|
|
24 | 24 | |
|
|
25 |
CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', ' |
|
|
|
25 | CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis', 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')] | |
|
|
26 | 26 | |
|
|
27 | 27 | |
|
|
28 | 28 | def figpause(interval): |
| @@ -45,8 +45,7 class PlotData(Operation, Process): | |||
|
|
45 | 45 | CODE = 'Figure' |
|
|
46 | 46 | colormap = 'jro' |
|
|
47 | 47 | bgcolor = 'white' |
|
|
48 | CONFLATE = False | |
|
|
49 | __MAXNUMX = 80 | |
|
|
48 | CONFLATE = False | |
|
|
50 | 49 | __missing = 1E30 |
|
|
51 | 50 | |
|
|
52 | 51 | __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax', |
| @@ -88,7 +87,7 class PlotData(Operation, Process): | |||
|
|
88 | 87 | self.ymin = kwargs.get('ymin', None) |
|
|
89 | 88 | self.ymax = kwargs.get('ymax', None) |
|
|
90 | 89 | self.xlabel = kwargs.get('xlabel', None) |
|
|
91 |
self. |
|
|
|
90 | self.decimation = kwargs.get('decimation', None) | |
|
|
92 | 91 | self.showSNR = kwargs.get('showSNR', False) |
|
|
93 | 92 | self.oneFigure = kwargs.get('oneFigure', True) |
|
|
94 | 93 | self.width = kwargs.get('width', None) |
| @@ -328,7 +327,7 class PlotData(Operation, Process): | |||
|
|
328 | 327 | def decimate(self): |
|
|
329 | 328 | |
|
|
330 | 329 | # dx = int(len(self.x)/self.__MAXNUMX) + 1 |
|
|
331 |
dy = int(len(self.y) / self. |
|
|
|
330 | dy = int(len(self.y) / self.decimation) + 1 | |
|
|
332 | 331 | |
|
|
333 | 332 | # x = self.x[::dx] |
|
|
334 | 333 | x = self.x |
| @@ -369,7 +368,7 class PlotData(Operation, Process): | |||
|
|
369 | 368 | ymin = self.ymin if self.ymin else numpy.nanmin(self.y) |
|
|
370 | 369 | ymax = self.ymax if self.ymax else numpy.nanmax(self.y) |
|
|
371 | 370 | |
|
|
372 | Y = numpy.array([10, 20, 50, 100, 200, 500, 1000, 2000]) | |
|
|
371 | Y = numpy.array([5, 10, 20, 50, 100, 200, 500, 1000, 2000]) | |
|
|
373 | 372 | i = 1 if numpy.where(ymax-ymin < Y)[0][0] < 0 else numpy.where(ymax-ymin < Y)[0][0] |
|
|
374 | 373 | ystep = Y[i] / 5 |
|
|
375 | 374 | |
| @@ -422,8 +421,11 class PlotData(Operation, Process): | |||
|
|
422 | 421 | ''' |
|
|
423 | 422 | log.success('Plotting', self.name) |
|
|
424 | 423 | |
|
|
425 | self.plot() | |
|
|
426 |
self. |
|
|
|
424 | try: | |
|
|
425 | self.plot() | |
|
|
426 | self.format() | |
|
|
427 | except: | |
|
|
428 | log.warning('{} Plot could not be updated... check data'.format(self.CODE), self.name) | |
|
|
427 | 429 | |
|
|
428 | 430 | for n, fig in enumerate(self.figures): |
|
|
429 | 431 | if self.nrows == 0 or self.nplots == 0: |
| @@ -449,7 +451,7 class PlotData(Operation, Process): | |||
|
|
449 | 451 | self.CODE, |
|
|
450 | 452 | label, |
|
|
451 | 453 | self.getDateTime(self.saveTime).strftime( |
|
|
452 |
'% |
|
|
|
454 | '%Y%m%d_%H%M%S'), | |
|
|
453 | 455 | ) |
|
|
454 | 456 | ) |
|
|
455 | 457 | log.log('Saving figure: {}'.format(figname), self.name) |
| @@ -526,7 +528,7 class PlotSpectraData(PlotData): | |||
|
|
526 | 528 | if self.showprofile: |
|
|
527 | 529 | self.width += 0.8 * self.ncols |
|
|
528 | 530 | |
|
|
529 |
self.ylabel = 'Range [ |
|
|
|
531 | self.ylabel = 'Range [km]' | |
|
|
530 | 532 | |
|
|
531 | 533 | def plot(self): |
|
|
532 | 534 | if self.xaxis == "frequency": |
| @@ -598,7 +600,7 class PlotCrossSpectraData(PlotData): | |||
|
|
598 | 600 | self.nplots = self.nrows * 4 |
|
|
599 | 601 | self.width = 3.4 * self.ncols |
|
|
600 | 602 | self.height = 3 * self.nrows |
|
|
601 |
self.ylabel = 'Range [ |
|
|
|
603 | self.ylabel = 'Range [km]' | |
|
|
602 | 604 | self.showprofile = False |
|
|
603 | 605 | |
|
|
604 | 606 | def plot(self): |
| @@ -701,7 +703,7 class PlotRTIData(PlotData): | |||
|
|
701 | 703 | self.ncols = 1 |
|
|
702 | 704 | self.nrows = len(self.data.channels) |
|
|
703 | 705 | self.nplots = len(self.data.channels) |
|
|
704 |
self.ylabel = 'Range [ |
|
|
|
706 | self.ylabel = 'Range [km]' | |
|
|
705 | 707 | self.cb_label = 'dB' |
|
|
706 | 708 | self.titles = ['{} Channel {}'.format( |
|
|
707 | 709 | self.CODE.upper(), x) for x in range(self.nrows)] |
| @@ -712,8 +714,12 class PlotRTIData(PlotData): | |||
|
|
712 | 714 | self.z = self.data[self.CODE] |
|
|
713 | 715 | self.z = numpy.ma.masked_invalid(self.z) |
|
|
714 | 716 | |
|
|
715 | for n, ax in enumerate(self.axes): | |
|
|
717 | if self.decimation is None: | |
|
|
718 | x, y, z = self.fill_gaps(self.x, self.y, self.z) | |
|
|
719 | else: | |
|
|
716 | 720 | x, y, z = self.fill_gaps(*self.decimate()) |
|
|
721 | ||
|
|
722 | for n, ax in enumerate(self.axes): | |
|
|
717 | 723 | self.zmin = self.zmin if self.zmin else numpy.min(self.z) |
|
|
718 | 724 | self.zmax = self.zmax if self.zmax else numpy.max(self.z) |
|
|
719 | 725 | if ax.firsttime: |
| @@ -754,7 +760,7 class PlotCOHData(PlotRTIData): | |||
|
|
754 | 760 | self.ncols = 1 |
|
|
755 | 761 | self.nrows = len(self.data.pairs) |
|
|
756 | 762 | self.nplots = len(self.data.pairs) |
|
|
757 |
self.ylabel = 'Range [ |
|
|
|
763 | self.ylabel = 'Range [km]' | |
|
|
758 | 764 | if self.CODE == 'coh': |
|
|
759 | 765 | self.cb_label = '' |
|
|
760 | 766 | self.titles = [ |
| @@ -896,7 +902,7 class PlotParamData(PlotRTIData): | |||
|
|
896 | 902 | self.nrows += 1 |
|
|
897 | 903 | self.nplots += 1 |
|
|
898 | 904 | |
|
|
899 |
self.ylabel = 'Height [ |
|
|
|
905 | self.ylabel = 'Height [km]' | |
|
|
900 | 906 | if not self.titles: |
|
|
901 | 907 | self.titles = self.data.parameters \ |
|
|
902 | 908 | if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)] |
| @@ -916,9 +922,13 class PlotParamData(PlotRTIData): | |||
|
|
916 | 922 | |
|
|
917 | 923 | self.z = numpy.ma.masked_invalid(self.z) |
|
|
918 | 924 | |
|
|
919 | for n, ax in enumerate(self.axes): | |
|
|
920 | ||
|
|
925 | if self.decimation is None: | |
|
|
926 | x, y, z = self.fill_gaps(self.x, self.y, self.z) | |
|
|
927 | else: | |
|
|
921 | 928 | x, y, z = self.fill_gaps(*self.decimate()) |
|
|
929 | ||
|
|
930 | for n, ax in enumerate(self.axes): | |
|
|
931 | ||
|
|
922 | 932 | self.zmax = self.zmax if self.zmax is not None else numpy.max( |
|
|
923 | 933 | self.z[n]) |
|
|
924 | 934 | self.zmin = self.zmin if self.zmin is not None else numpy.min( |
| @@ -1,13 +1,19 | |||
|
|
1 |
import |
|
|
|
2 | import datetime | |
|
|
1 | import os | |
|
|
3 | 2 | import sys |
|
|
3 | import datetime | |
|
|
4 | import numpy | |
|
|
4 | 5 | import matplotlib |
|
|
5 | 6 | |
|
|
6 | if 'linux' in sys.platform: | |
|
|
7 | if 'BACKEND' in os.environ: | |
|
|
8 | matplotlib.use(os.environ['BACKEND']) | |
|
|
9 | elif 'linux' in sys.platform: | |
|
|
7 | 10 | matplotlib.use("TkAgg") |
|
|
8 | ||
|
|
9 | if 'darwin' in sys.platform: | |
|
|
10 | matplotlib.use('TKAgg') | |
|
|
11 | elif 'darwin' in sys.platform: | |
|
|
12 | matplotlib.use('TkAgg') | |
|
|
13 | else: | |
|
|
14 | from schainpy.utils import log | |
|
|
15 | log.warning('Using default Backend="Agg"', 'INFO') | |
|
|
16 | matplotlib.use('Agg') | |
|
|
11 | 17 | # Qt4Agg', 'GTK', 'GTKAgg', 'ps', 'agg', 'cairo', 'MacOSX', 'GTKCairo', 'WXAgg', 'template', 'TkAgg', 'GTK3Cairo', 'GTK3Agg', 'svg', 'WebAgg', 'CocoaAgg', 'emf', 'gdk', 'WX' |
|
|
12 | 18 | import matplotlib.pyplot |
|
|
13 | 19 | |
| @@ -22,6 +22,7 except: | |||
|
|
22 | 22 | |
|
|
23 | 23 | from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader |
|
|
24 | 24 | from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width |
|
|
25 | from schainpy.utils import log | |
|
|
25 | 26 | |
|
|
26 | 27 | LOCALTIME = True |
|
|
27 | 28 | |
| @@ -440,7 +441,7 def isRadarFolder(folder): | |||
|
|
440 | 441 | |
|
|
441 | 442 | |
|
|
442 | 443 | def isRadarFile(file): |
|
|
443 | try: | |
|
|
444 | try: | |
|
|
444 | 445 | year = int(file[1:5]) |
|
|
445 | 446 | doy = int(file[5:8]) |
|
|
446 | 447 | set = int(file[8:11]) |
| @@ -451,10 +452,10 def isRadarFile(file): | |||
|
|
451 | 452 | |
|
|
452 | 453 | |
|
|
453 | 454 | def getDateFromRadarFile(file): |
|
|
454 | try: | |
|
|
455 | try: | |
|
|
455 | 456 | year = int(file[1:5]) |
|
|
456 | 457 | doy = int(file[5:8]) |
|
|
457 | set = int(file[8:11]) | |
|
|
458 | set = int(file[8:11]) | |
|
|
458 | 459 | except: |
|
|
459 | 460 | return None |
|
|
460 | 461 | |
| @@ -649,26 +650,13 class JRODataReader(JRODataIO): | |||
|
|
649 | 650 | |
|
|
650 | 651 | filenameList = [] |
|
|
651 | 652 | datetimeList = [] |
|
|
652 | ||
|
|
653 | ||
|
|
653 | 654 | for thisPath in pathList: |
|
|
654 | 655 | |
|
|
655 | 656 | fileList = glob.glob1(thisPath, "*%s" % ext) |
|
|
656 | 657 | fileList.sort() |
|
|
657 | 658 | |
|
|
658 |
|
|
|
|
659 | ||
|
|
660 | if cursor is not None and skip is not None: | |
|
|
661 | ||
|
|
662 | if skip == 0: | |
|
|
663 | skippedFileList = [] | |
|
|
664 | else: | |
|
|
665 | skippedFileList = fileList[cursor * | |
|
|
666 | skip: cursor * skip + skip] | |
|
|
667 | ||
|
|
668 | else: | |
|
|
669 | skippedFileList = fileList | |
|
|
670 | ||
|
|
671 | for file in skippedFileList: | |
|
|
659 | for file in fileList: | |
|
|
672 | 660 | |
|
|
673 | 661 | filename = os.path.join(thisPath, file) |
|
|
674 | 662 | |
| @@ -684,12 +672,15 class JRODataReader(JRODataIO): | |||
|
|
684 | 672 | filenameList.append(filename) |
|
|
685 | 673 | datetimeList.append(thisDatetime) |
|
|
686 | 674 | |
|
|
675 | if cursor is not None and skip is not None: | |
|
|
676 | filenameList = filenameList[cursor * skip:cursor * skip + skip] | |
|
|
677 | datetimeList = datetimeList[cursor * skip:cursor * skip + skip] | |
|
|
678 | ||
|
|
687 | 679 | if not(filenameList): |
|
|
688 | 680 | print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path) |
|
|
689 | 681 | return [], [] |
|
|
690 | 682 | |
|
|
691 | print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime) | |
|
|
692 | ||
|
|
683 | print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime) | |
|
|
693 | 684 | |
|
|
694 | 685 | # for i in range(len(filenameList)): |
|
|
695 | 686 | # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) |
| @@ -1743,9 +1734,9 class JRODataWriter(JRODataIO): | |||
|
|
1743 | 1734 | if self.dataOut.datatime.date() > self.fileDate: |
|
|
1744 | 1735 | setFile = 0 |
|
|
1745 | 1736 | self.nTotalBlocks = 0 |
|
|
1746 | ||
|
|
1747 | filen = '%s%4.4d%3.3d%3.3d%s' % ( | |
|
|
1748 | self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext) | |
|
|
1737 | ||
|
|
1738 | filen = '{}{:04d}{:03d}{:03d}{}'.format( | |
|
|
1739 | self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext) | |
|
|
1749 | 1740 | |
|
|
1750 | 1741 | filename = os.path.join(path, subfolder, filen) |
|
|
1751 | 1742 | |
| @@ -1796,11 +1787,11 class JRODataWriter(JRODataIO): | |||
|
|
1796 | 1787 | self.ext = ext.lower() |
|
|
1797 | 1788 | |
|
|
1798 | 1789 | self.path = path |
|
|
1799 | ||
|
|
1790 | ||
|
|
1800 | 1791 | if set is None: |
|
|
1801 | 1792 | self.setFile = -1 |
|
|
1802 | 1793 | else: |
|
|
1803 | self.setFile = set - 1 | |
|
|
1794 | self.setFile = set - 1 | |
|
|
1804 | 1795 | |
|
|
1805 | 1796 | self.blocksPerFile = blocksPerFile |
|
|
1806 | 1797 | |
| @@ -82,9 +82,10 class Data(object): | |||
|
|
82 | 82 | Object to hold data to be plotted |
|
|
83 | 83 | ''' |
|
|
84 | 84 | |
|
|
85 | def __init__(self, plottypes, throttle_value): | |
|
|
85 | def __init__(self, plottypes, throttle_value, exp_code): | |
|
|
86 | 86 | self.plottypes = plottypes |
|
|
87 | 87 | self.throttle = throttle_value |
|
|
88 | self.exp_code = exp_code | |
|
|
88 | 89 | self.ended = False |
|
|
89 | 90 | self.localtime = False |
|
|
90 | 91 | self.__times = [] |
| @@ -147,7 +148,8 class Data(object): | |||
|
|
147 | 148 | return |
|
|
148 | 149 | |
|
|
149 | 150 | self.parameters = getattr(dataOut, 'parameters', []) |
|
|
150 |
|
|
|
|
151 | if hasattr(dataOut, 'pairsList'): | |
|
|
152 | self.pairs = dataOut.pairsList | |
|
|
151 | 153 | self.channels = dataOut.channelList |
|
|
152 | 154 | self.interval = dataOut.getTimeInterval() |
|
|
153 | 155 | self.localtime = dataOut.useLocalTime |
| @@ -214,17 +216,28 class Data(object): | |||
|
|
214 | 216 | Convert data to json |
|
|
215 | 217 | ''' |
|
|
216 | 218 | |
|
|
217 |
|
|
|
|
219 | data = {} | |
|
|
218 | 220 | tm = self.times[-1] |
|
|
219 | ||
|
|
220 |
for key |
|
|
|
221 | ||
|
|
222 | for key in self.data: | |
|
|
221 | 223 | if key in ('spc', 'cspc'): |
|
|
222 |
|
|
|
|
224 | dx = int(self.data[key].shape[1]/MAXNUMX) + 1 | |
|
|
225 | dy = int(self.data[key].shape[2]/MAXNUMY) + 1 | |
|
|
226 | data[key] = roundFloats(self.data[key][::, ::dx, ::dy].tolist()) | |
|
|
223 | 227 | else: |
|
|
224 |
|
|
|
|
228 | data[key] = roundFloats(self.data[key][tm].tolist()) | |
|
|
225 | 229 | |
|
|
226 | ret['timestamp'] = tm | |
|
|
230 | ret = {'data': data} | |
|
|
231 | ret['exp_code'] = self.exp_code | |
|
|
232 | ret['time'] = tm | |
|
|
227 | 233 | ret['interval'] = self.interval |
|
|
234 | ret['localtime'] = self.localtime | |
|
|
235 | ret['yrange'] = roundFloats(self.heights.tolist()) | |
|
|
236 | if key in ('spc', 'cspc'): | |
|
|
237 | ret['xrange'] = roundFloats(self.xrange[2][::dx].tolist()) | |
|
|
238 | if hasattr(self, 'pairs'): | |
|
|
239 | ret['pairs'] = self.pairs | |
|
|
240 | return json.dumps(ret) | |
|
|
228 | 241 | |
|
|
229 | 242 | @property |
|
|
230 | 243 | def times(self): |
| @@ -475,7 +488,8 class ReceiverData(ProcessingUnit): | |||
|
|
475 | 488 | class PlotterReceiver(ProcessingUnit, Process): |
|
|
476 | 489 | |
|
|
477 | 490 | throttle_value = 5 |
|
|
478 |
__attrs__ = ['server', 'plottypes', 'realtime', 'localtime', 'throttle' |
|
|
|
491 | __attrs__ = ['server', 'plottypes', 'realtime', 'localtime', 'throttle', | |
|
|
492 | 'exp_code', 'web_server'] | |
|
|
479 | 493 | |
|
|
480 | 494 | def __init__(self, **kwargs): |
|
|
481 | 495 | |
| @@ -486,30 +500,25 class PlotterReceiver(ProcessingUnit, Process): | |||
|
|
486 | 500 | self.isWebConfig = False |
|
|
487 | 501 | self.connections = 0 |
|
|
488 | 502 | server = kwargs.get('server', 'zmq.pipe') |
|
|
489 |
|
|
|
|
503 | web_server = kwargs.get('web_server', None) | |
|
|
490 | 504 | if 'tcp://' in server: |
|
|
491 | 505 | address = server |
|
|
492 | 506 | else: |
|
|
493 | 507 | address = 'ipc:///tmp/%s' % server |
|
|
494 | ||
|
|
495 | if 'tcp://' in plot_server: | |
|
|
496 | plot_address = plot_server | |
|
|
497 | else: | |
|
|
498 | plot_address = 'ipc:///tmp/%s' % plot_server | |
|
|
499 | ||
|
|
500 | 508 | self.address = address |
|
|
501 |
self. |
|
|
|
509 | self.web_address = web_server | |
|
|
502 | 510 | self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')] |
|
|
503 | 511 | self.realtime = kwargs.get('realtime', False) |
|
|
504 | 512 | self.localtime = kwargs.get('localtime', True) |
|
|
505 | 513 | self.throttle_value = kwargs.get('throttle', 5) |
|
|
514 | self.exp_code = kwargs.get('exp_code', None) | |
|
|
506 | 515 | self.sendData = self.initThrottle(self.throttle_value) |
|
|
507 | 516 | self.dates = [] |
|
|
508 | 517 | self.setup() |
|
|
509 | 518 | |
|
|
510 | 519 | def setup(self): |
|
|
511 | 520 | |
|
|
512 | self.data = Data(self.plottypes, self.throttle_value) | |
|
|
521 | self.data = Data(self.plottypes, self.throttle_value, self.exp_code) | |
|
|
513 | 522 | self.isConfig = True |
|
|
514 | 523 | |
|
|
515 | 524 | def event_monitor(self, monitor): |
| @@ -559,9 +568,13 class PlotterReceiver(ProcessingUnit, Process): | |||
|
|
559 | 568 | self.receiver.bind(self.address) |
|
|
560 | 569 | monitor = self.receiver.get_monitor_socket() |
|
|
561 | 570 | self.sender = self.context.socket(zmq.PUB) |
|
|
562 |
if self. |
|
|
|
571 | if self.web_address: | |
|
|
572 | log.success( | |
|
|
573 | 'Sending to web: {}'.format(self.web_address), | |
|
|
574 | self.name | |
|
|
575 | ) | |
|
|
563 | 576 | self.sender_web = self.context.socket(zmq.PUB) |
|
|
564 |
self.sender_web.connect(self. |
|
|
|
577 | self.sender_web.connect(self.web_address) | |
|
|
565 | 578 | time.sleep(1) |
|
|
566 | 579 | |
|
|
567 | 580 | if 'server' in self.kwargs: |
| @@ -609,27 +622,10 class PlotterReceiver(ProcessingUnit, Process): | |||
|
|
609 | 622 | else: |
|
|
610 | 623 | if self.realtime: |
|
|
611 | 624 | self.send(self.data) |
|
|
612 | # self.sender_web.send_string(self.data.jsonify()) | |
|
|
625 | if self.web_address: | |
|
|
626 | self.sender_web.send(self.data.jsonify()) | |
|
|
613 | 627 | else: |
|
|
614 | 628 | self.sendData(self.send, self.data, coerce=coerce) |
|
|
615 | 629 | coerce = False |
|
|
616 | 630 | |
|
|
617 | 631 | return |
|
|
618 | ||
|
|
619 | def sendToWeb(self): | |
|
|
620 | ||
|
|
621 | if not self.isWebConfig: | |
|
|
622 | context = zmq.Context() | |
|
|
623 | sender_web_config = context.socket(zmq.PUB) | |
|
|
624 | if 'tcp://' in self.plot_address: | |
|
|
625 | dum, address, port = self.plot_address.split(':') | |
|
|
626 | conf_address = '{}:{}:{}'.format(dum, address, int(port)+1) | |
|
|
627 | else: | |
|
|
628 | conf_address = self.plot_address + '.config' | |
|
|
629 | sender_web_config.bind(conf_address) | |
|
|
630 | time.sleep(1) | |
|
|
631 | for kwargs in self.operationKwargs.values(): | |
|
|
632 | if 'plot' in kwargs: | |
|
|
633 | log.success('[Sending] Config data to web for {}'.format(kwargs['code'].upper())) | |
|
|
634 | sender_web_config.send_string(json.dumps(kwargs)) | |
|
|
635 | self.isWebConfig = True | |
| @@ -8,8 +8,6 import os | |||
|
|
8 | 8 | from setuptools import setup, Extension |
|
|
9 | 9 | from setuptools.command.build_ext import build_ext as _build_ext |
|
|
10 | 10 | from schainpy import __version__ |
|
|
11 | from schainpy.utils import log | |
|
|
12 | ||
|
|
13 | 11 | |
|
|
14 | 12 | class build_ext(_build_ext): |
|
|
15 | 13 | def finalize_options(self): |
| @@ -19,22 +17,13 class build_ext(_build_ext): | |||
|
|
19 | 17 | import numpy |
|
|
20 | 18 | self.include_dirs.append(numpy.get_include()) |
|
|
21 | 19 | |
|
|
22 | ||
|
|
23 | try: | |
|
|
24 | from PyQt4 import QtCore, QtGui | |
|
|
25 | from PyQt4.QtGui import QApplication | |
|
|
26 | except: | |
|
|
27 | log.warning( | |
|
|
28 | 'You should install PyQt4 module in order to run the GUI. See the README.') | |
|
|
29 | ||
|
|
30 | ||
|
|
31 | setup(name="schainpy", | |
|
|
32 | version=__version__, | |
|
|
33 | description="Python tools to read, write and process Jicamarca data", | |
|
|
34 | author="Miguel Urco", | |
|
|
35 | author_email="miguel.urco@jro.igp.gob.pe", | |
|
|
36 | url="http://jro.igp.gob.pe", | |
|
|
37 | packages={'schainpy', | |
|
|
20 | setup(name = "schainpy", | |
|
|
21 | version = __version__, | |
|
|
22 | description = "Python tools to read, write and process Jicamarca data", | |
|
|
23 | author = "Miguel Urco", | |
|
|
24 | author_email = "miguel.urco@jro.igp.gob.pe", | |
|
|
25 | url = "http://jro.igp.gob.pe", | |
|
|
26 | packages = {'schainpy', | |
|
|
38 | 27 | 'schainpy.model', |
|
|
39 | 28 | 'schainpy.model.data', |
|
|
40 | 29 | 'schainpy.model.graphics', |
| @@ -42,32 +31,33 setup(name="schainpy", | |||
|
|
42 | 31 | 'schainpy.model.proc', |
|
|
43 | 32 | 'schainpy.model.serializer', |
|
|
44 | 33 | 'schainpy.model.utils', |
|
|
34 | 'schainpy.utils', | |
|
|
45 | 35 | 'schainpy.gui', |
|
|
46 | 36 | 'schainpy.gui.figures', |
|
|
47 | 37 | 'schainpy.gui.viewcontroller', |
|
|
48 | 38 | 'schainpy.gui.viewer', |
|
|
49 |
'schainpy.gui.viewer.windows' |
|
|
|
50 | ext_package='schainpy', | |
|
|
51 | py_modules=[''], | |
|
|
52 | package_data={'': ['schain.conf.template'], | |
|
|
53 | 'schainpy.gui.figures': ['*.png', '*.jpg'], | |
|
|
54 | }, | |
|
|
55 | include_package_data=False, | |
|
|
56 | scripts=['schainpy/gui/schainGUI'], | |
|
|
57 | ext_modules=[ | |
|
|
58 | Extension("cSchain", ["schainpy/model/proc/extensions.c"] | |
|
|
59 |
|
|
|
|
60 | entry_points={ | |
|
|
39 | 'schainpy.gui.viewer.windows', | |
|
|
40 | 'schainpy.cli'}, | |
|
|
41 | ext_package = 'schainpy', | |
|
|
42 | package_data = {'': ['schain.conf.template'], | |
|
|
43 | 'schainpy.gui.figures': ['*.png', '*.jpg'], | |
|
|
44 | }, | |
|
|
45 | include_package_data = False, | |
|
|
46 | scripts = ['schainpy/gui/schainGUI'], | |
|
|
47 | ext_modules = [ | |
|
|
48 | Extension("cSchain", ["schainpy/model/proc/extensions.c"]) | |
|
|
49 | ], | |
|
|
50 | entry_points = { | |
|
|
61 | 51 | 'console_scripts': [ |
|
|
62 | 'schain = schaincli.cli:main', | |
|
|
52 | 'schain = schainpy.cli.cli:main', | |
|
|
63 | 53 | ], |
|
|
64 | 54 | }, |
|
|
65 | cmdclass={'build_ext': build_ext}, | |
|
|
66 | setup_requires=["numpy >= 1.11.2"], | |
|
|
67 | install_requires=[ | |
|
|
55 | cmdclass = {'build_ext': build_ext}, | |
|
|
56 | setup_requires = ["numpy >= 1.11.2"], | |
|
|
57 | install_requires = [ | |
|
|
68 | 58 | "scipy >= 0.14.0", |
|
|
69 | 59 | "h5py >= 2.2.1", |
|
|
70 |
"matplotlib >= |
|
|
|
60 | "matplotlib >= 2.0.0", | |
|
|
71 | 61 | "pyfits >= 3.4", |
|
|
72 | 62 | "paramiko >= 2.1.2", |
|
|
73 | 63 | "paho-mqtt >= 1.2", |
| @@ -75,5 +65,5 setup(name="schainpy", | |||
|
|
75 | 65 | "fuzzywuzzy", |
|
|
76 | 66 | "click", |
|
|
77 | 67 | "python-Levenshtein" |
|
|
78 | ], | |
|
|
79 | ) | |
|
|
68 | ], | |
|
|
69 | ) | |
General Comments 0
You need to be logged in to leave comments.
Login now