From ecbe1f26c35b490bef6ae0f21cd2bdd8af9774cc 2017-09-29 21:43:37 From: jespinoza Date: 2017-09-29 21:43:37 Subject: [PATCH] merge with v2.3 --- diff --git a/.gitignore b/.gitignore index 0a065d6..e257102 100644 --- a/.gitignore +++ b/.gitignore @@ -105,9 +105,11 @@ ENV/ .vscode +schainpy/scripts/ +schaingui/node_modules/ .svn/ *.png *.pyc -schainpy/scripts - -schaingui/node_modules +*.xml +*.log +trash \ No newline at end of file diff --git a/README.md b/README.md index 5704c9f..27e4544 100644 --- a/README.md +++ b/README.md @@ -8,24 +8,39 @@ Signal Chain (SCh) is a radar data processing library developed using [Python](w Install system dependencies, clone the latest version from [git](http://jro-dev.igp.gob.pe/rhodecode/schain/) and install it as a normal python package. +### Linux based system ``` $ sudo apt-get install python-pip python-dev gfortran libpng-dev freetype* libblas-dev liblapack-dev libatlas-base-dev python-qt4 python-tk libssl-dev libhdf5-dev $ sudo pip install numpy $ git clone http://jro-dev.igp.gob.pe/rhodecode/schain/ $ cd schain $ sudo pip install ./ -``` - -**Its recommended to install schain in a virtual environment** ``` +**It is recommended to install schain in a virtual environment** +``` $ sudo pip install virtualenv $ virtualenv /path/to/virtual --system-site-packages $ source /path/to/virtual/bin/activate (virtual) $ cd schain (virtual) $ pip install ./ + ``` +### MAC Os +``` +$ brew install cartr/qt4/pyqt +$ git clone http://jro-dev.igp.gob.pe/rhodecode/schain/ +$ cd schain +$ pip install ./ +``` + +if ```pip install ./``` does not work, install a proper python enviroment, and repeat the steps. +``` +$ brew install python +``` + + ## First Script Read Spectra data (.pdata) - remove dc - plot spectra & RTI diff --git a/schaincli/cli.py b/schaincli/cli.py index fdf7531..5b03de2 100644 --- a/schaincli/cli.py +++ b/schaincli/cli.py @@ -8,12 +8,13 @@ save_stdout = sys.stdout sys.stdout = open('trash', 'w') from multiprocessing import cpu_count from schaincli import templates -from schainpy import controller_api +from schainpy.controller import Project from schainpy.model import Operation, ProcessingUnit from schainpy.utils import log from importlib import import_module from pydoc import locate from fuzzywuzzy import process +from schainpy.utils import paramsFinder sys.stdout = save_stdout @@ -75,47 +76,25 @@ def search(nextcommand): if nextcommand is None: log.error('There is no Operation/ProcessingUnit to search') elif nextcommand == 'procs': - module = dir(import_module('schainpy.model')) - procs = check_module(module, ProcessingUnit) - try: - procs.remove('ProcessingUnit') - except Exception as e: - pass + procs = paramsFinder.getProcs() log.success('Current ProcessingUnits are:\n\033[1m{}\033[0m'.format('\n'.join(procs))) elif nextcommand == 'operations': - module = dir(import_module('schainpy.model')) - noProcs = [x for x in module if not x.endswith('Proc')] - operations = check_module(noProcs, Operation) - try: - operations.remove('Operation') - except Exception as e: - pass + operations = paramsFinder.getOperations() log.success('Current Operations are:\n\033[1m{}\033[0m'.format('\n'.join(operations))) else: try: - module = locate('schainpy.model.{}'.format(nextcommand)) - args = module().getAllowedArgs() + args = paramsFinder.getArgs(nextcommand) log.warning('Use this feature with caution. It may not return all the allowed arguments') - try: - args.remove('self') - except Exception as e: - pass - try: - args.remove('dataOut') - except Exception as e: - pass if len(args) == 0: log.success('{} has no arguments'.format(nextcommand)) else: log.success('Showing arguments of {} are:\n\033[1m{}\033[0m'.format(nextcommand, '\n'.join(args))) except Exception as e: log.error('Module {} does not exists'.format(nextcommand)) - allModules = dir(import_module('schainpy.model')) - module = check_module(allModules, Operation) - module.extend(check_module(allModules, ProcessingUnit)) - similar = process.extractOne(nextcommand, module)[0] - log.success('Searching {} instead'.format(similar)) + allModules = paramsFinder.getAll() + similar = process.extractOne(nextcommand, allModules)[0] + log.success('Showing {} instead'.format(similar)) search(similar) @@ -171,18 +150,8 @@ def test(): def runFromXML(filename): - controller = controller_api.ControllerThread() + controller = Project() if not controller.readXml(filename): return - - plotterObj = controller.useExternalPlotter() - controller.start() - plotterObj.start() - - cliLogger("Finishing all processes") - - controller.join(5) - - cliLogger("End of script") return diff --git a/schainpy/CHANGELOG.md b/schainpy/CHANGELOG.md new file mode 100644 index 0000000..e2ae29a --- /dev/null +++ b/schainpy/CHANGELOG.md @@ -0,0 +1,109 @@ +## CHANGELOG: + +### 2.3 +* Added high order function `multiSchain` for multiprocessing scripts. +* Added two new Processing Units `PublishData` and `ReceiverData` for receiving and sending dataOut through multiple ways (tcp, ipc, inproc). +* Added a new graphics Processing Unit `PlotterReceiver`. It is decoupled from normal processing sequence with support for data generated by multiprocessing scripts. +* Added support for sending realtime graphic to web server. +* GUI command `schain` is now `schainGUI`. +* Added a CLI tool named `schain`. + * Scripts templates can be now generated with `schain generate`. + * Now it is possible to search Processing Units and Operations with `schain search [module]` to get the right name and its allowed parameters. + * `schain xml` to run xml scripts. +* Added suggestions when parameters are poorly written. +* `Controller.start()` now runs in a different process than the process calling it. +* Added `schainpy.utils.log` for log standarization. +* Running script on online mode no longer ignores date and hour. Issue #1109. +* Added support for receving voltage data directly from JARS (tcp, ipc). +* Updated README for MAC OS GUI installation. +* Setup now installs numpy. + +### 2.2.6 +* Graphics generated by the GUI are now the same as generated by scripts. Issue #1074. +* Added support for C extensions. +* function `hildebrand_sehkon` optimized with a C wrapper. +* Numpy version updated. +* Migration to GIT. + +### 2.2.5: +* splitProfiles and combineProfiles modules were added to VoltageProc and Signal Chain GUI. +* nProfiles of USRP data (hdf5) is the number of profiles thera are in one second. +* jroPlotter works directly with data objects instead of dictionaries +* script "schain" was added to Signal Chain installer + +### 2.2.4.1: +* jroIO_usrp.py is update to read Sandra's data +* decimation in Spectra and RTI plots is always enabled. +* time* window option added to GUI + +### 2.2.4: +* jroproc_spectra_lags.py added to schainpy +* Bug fixed in schainGUI: ProcUnit was created with the same id in some cases. +* Bug fixed in jroHeaderIO: Header size validation. + +### 2.2.3.1: +* Filtering block by time has been added. +* Bug fixed plotting RTI, CoherenceMap and others using xmin and xmax parameters. The first day worked +properly but the next days did not. + +### 2.2.3: +* Bug fixed in GUI: Error getting(reading) Code value +* Bug fixed in GUI: Flip option always needs channelList field +* Bug fixed in jrodata: when one branch modified a value in "dataOut" (example: dataOut.code) this value +was modified for every branch (because this was a reference). It was modified in data.copy() +* Bug fixed in jroproc_voltage.profileSelector(): rangeList replaces to profileRangeList. + +### 2.2.2: +* VoltageProc: ProfileSelector, Reshape, Decoder with nTxs!=1 and getblock=True was tested +* Rawdata and testRawdata.py added to Signal Chain project + +### 2.2.1: +* Bugs fixed in GUI +* Views were improved in GUI +* Support to MST* ISR experiments +* Bug fixed getting noise using hyldebrant. (minimum number of points > 20%) +* handleError added to jroplotter.py + +### 2.2.0: +* GUI: use of external plotter +* Compatible with matplotlib 1.5.0 + +### 2.1.5: +* serializer module added to Signal Chain +* jroplotter.py added to Signal Chain + +### 2.1.4.2: +* A new Plotter Class was added +* Project.start() does not accept filename as a parameter anymore + +### 2.1.4.1: +* Send notifications when an error different to ValueError is detected + +### 2.1.4: +* Sending error notifications to signal chain administrator +* Login to email server added + +### 2.1.3.3: +* Colored Button Icons were added to GUI + +### 2.1.3.2: +* GUI: user interaction enhanced +* controller_api.py: Safe access to ControllerThead + +### 2.1.3.1: +* GUI: every icon were resized +* jroproc_voltage.py: Print a message when "Read from code" option is selected and the code is not defined inside data file + +### 2.1.3: +* jroplot_heispectra.py: SpectraHeisScope was not showing the right channels +* jroproc_voltage.py: Bug fixed selecting profiles (self.nProfiles took a wrong value), + Bug fixed selecting heights by block (selecting profiles instead heights) +* jroproc_voltage.py: New feature added: decoding data by block using FFT. +* jroIO_heispectra.py: Bug fixed in FitsReader. Using local Fits instance instead schainpy.mode.data.jrodata.Fits. +* jroIO_heispectra.py: Channel index list does not exist. + +### 2.1.2: +* jroutils_ftp.py: Bug fixed, Any error sending file stopped the Server Thread + Server thread opens and closes remote server each time file list is sent +* jroplot_spectra.py: Noise path was not being created when noise data is saved. +* jroIO_base.py: startTime can be greater than endTime. Example: SpreadF [18:00 * 07:00] \ No newline at end of file diff --git a/schainpy/VERSION b/schainpy/VERSION deleted file mode 100644 index a19b1a9..0000000 --- a/schainpy/VERSION +++ /dev/null @@ -1,85 +0,0 @@ -VERSIONS: - -2.1.2: --jroutils_ftp.py: Bug fixed, Any error sending file stopped the Server Thread - Server thread opens and closes remote server each time file list is sent --jroplot_spectra.py: Noise path was not being created when noise data is saved. --jroIO_base.py: startTime can be greater than endTime. Example: SpreadF [18:00 - 07:00] - -2.1.3: --jroplot_heispectra.py: SpectraHeisScope was not showing the right channels --jroproc_voltage.py: Bug fixed selecting profiles (self.nProfiles took a wrong value), - Bug fixed selecting heights by block (selecting profiles instead heights) --jroproc_voltage.py: New feature added: decoding data by block using FFT. --jroIO_heispectra.py: Bug fixed in FitsReader. Using local Fits instance instead schainpy.mode.data.jrodata.Fits. --jroIO_heispectra.py: Channel index list does not exist. - -2.1.3.1: --GUI: every icon were resized --jroproc_voltage.py: Print a message when "Read from code" option is selected and the code is not defined inside data file - -2.1.3.2: --GUI: user interaction enhanced --controller_api.py: Safe access to ControllerThead - -2.1.3.3: --Colored Button Icons were added to GUI - -2.1.4: --Sending error notifications to signal chain administrator --Login to email server added - -2.1.4.1: --Send notifications when an error different to ValueError is detected - -2.1.4.2: --A new Plotter Class was added --Project.start() does not accept filename as a parameter anymore - -2.1.5: --serializer module added to Signal Chain --jroplotter.py added to Signal Chain - -2.2.0: --GUI: use of external plotter --Compatible with matplotlib 1.5.0 - -2.2.1: --Bugs fixed in GUI --Views were improved in GUI --Support to MST-ISR experiments --Bug fixed getting noise using hyldebrant. (minimum number of points > 20%) --handleError added to jroplotter.py - -2.2.2: --VoltageProc: ProfileSelector, Reshape, Decoder with nTxs!=1 and getblock=True was tested --Rawdata and testRawdata.py added to Signal Chain project - -2.2.3: --Bug fixed in GUI: Error getting(reading) Code value --Bug fixed in GUI: Flip option always needs channelList field --Bug fixed in jrodata: when one branch modified a value in "dataOut" (example: dataOut.code) this value -was modified for every branch (because this was a reference). It was modified in data.copy() --Bug fixed in jroproc_voltage.profileSelector(): rangeList replaces to profileRangeList. - - -2.2.3.1: --Filtering block by time has been added. --Bug fixed plotting RTI, CoherenceMap and others using xmin and xmax parameters. The first day worked -properly but the next days did not. - -2.2.4: --jroproc_spectra_lags.py added to schainpy --Bug fixed in schainGUI: ProcUnit was created with the same id in some cases. --Bug fixed in jroHeaderIO: Header size validation. - -2.2.4.1: --jroIO_usrp.py is update to read Sandra's data --decimation in Spectra and RTI plots is always enabled. --time-window option added to GUI - -2.2.5: --splitProfiles and combineProfiles modules were added to VoltageProc and Signal Chain GUI. --nProfiles of USRP data (hdf5) is the number of profiles thera are in one second. --jroPlotter works directly with data objects instead of dictionaries --script "schain" was added to Signal Chain installer \ No newline at end of file diff --git a/schainpy/__init__.py b/schainpy/__init__.py index 3b8fe16..d46a2d0 100644 --- a/schainpy/__init__.py +++ b/schainpy/__init__.py @@ -4,4 +4,4 @@ Created on Feb 7, 2012 @author $Author$ @version $Id$ ''' -__version__ = "2.3" \ No newline at end of file +__version__ = "2.3" diff --git a/schainpy/controller.py b/schainpy/controller.py index f642109..412bc2d 100644 --- a/schainpy/controller.py +++ b/schainpy/controller.py @@ -9,53 +9,53 @@ import datetime import traceback import math import time -from multiprocessing import Process, Queue, cpu_count - -import schainpy -import schainpy.admin +from multiprocessing import Process, cpu_count from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring from xml.dom import minidom +import schainpy +import schainpy.admin from schainpy.model import * -from time import sleep - -def prettify(elem): - """Return a pretty-printed XML string for the Element. - """ - rough_string = tostring(elem, 'utf-8') - reparsed = minidom.parseString(rough_string) - return reparsed.toprettyxml(indent=" ") - -def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False): - skip = 0 - cursor = 0 - nFiles = None - processes = [] - dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d') - dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d') +from schainpy.utils import log + +DTYPES = { + 'Voltage': '.r', + 'Spectra': '.pdata' +} + +def MPProject(project, n=cpu_count()): + ''' + Project wrapper to run schain in n processes + ''' + + rconf = project.getReadUnitObj() + op = rconf.getOperationObj('run') + dt1 = op.getParameterValue('startDate') + dt2 = op.getParameterValue('endDate') days = (dt2 - dt1).days - + for day in range(days+1): skip = 0 cursor = 0 - q = Queue() processes = [] - dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d') - firstProcess = Process(target=child, args=(cursor, skip, q, dt)) - firstProcess.start() - if by_day: - continue - nFiles = q.get() - if nFiles==0: + dt = dt1 + datetime.timedelta(day) + dt_str = dt.strftime('%Y/%m/%d') + reader = JRODataReader() + paths, files = reader.searchFilesOffLine(path=rconf.path, + startDate=dt, + endDate=dt, + ext=DTYPES[rconf.datatype]) + nFiles = len(files) + if nFiles == 0: continue - firstProcess.terminate() - skip = int(math.ceil(nFiles/nProcess)) - while True: - processes.append(Process(target=child, args=(cursor, skip, q, dt))) - processes[cursor].start() - if nFiles < cursor*skip: - break + skip = int(math.ceil(nFiles/n)) + while nFiles > cursor*skip: + rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor, + skip=skip) + p = project.clone() + p.start() + processes.append(p) cursor += 1 def beforeExit(exctype, value, trace): @@ -72,7 +72,6 @@ def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_da time.sleep(3) - class ParameterConf(): id = None @@ -109,7 +108,7 @@ class ParameterConf(): return self.__formated_value if value == '': - raise ValueError, "%s: This parameter value is empty" %self.name + raise ValueError, '%s: This parameter value is empty' %self.name if format == 'list': strList = value.split(',') @@ -119,10 +118,10 @@ class ParameterConf(): return self.__formated_value if format == 'intlist': - """ + ''' Example: value = (0,1,2) - """ + ''' new_value = ast.literal_eval(value) @@ -134,10 +133,10 @@ class ParameterConf(): return self.__formated_value if format == 'floatlist': - """ + ''' Example: value = (0.5, 1.4, 2.7) - """ + ''' new_value = ast.literal_eval(value) @@ -167,38 +166,38 @@ class ParameterConf(): return self.__formated_value if format == 'pairslist': - """ + ''' Example: value = (0,1),(1,2) - """ + ''' new_value = ast.literal_eval(value) if type(new_value) not in (tuple, list): - raise ValueError, "%s has to be a tuple or list of pairs" %value + raise ValueError, '%s has to be a tuple or list of pairs' %value if type(new_value[0]) not in (tuple, list): if len(new_value) != 2: - raise ValueError, "%s has to be a tuple or list of pairs" %value + raise ValueError, '%s has to be a tuple or list of pairs' %value new_value = [new_value] for thisPair in new_value: if len(thisPair) != 2: - raise ValueError, "%s has to be a tuple or list of pairs" %value + raise ValueError, '%s has to be a tuple or list of pairs' %value self.__formated_value = new_value return self.__formated_value if format == 'multilist': - """ + ''' Example: value = (0,1,2),(3,4,5) - """ + ''' multiList = ast.literal_eval(value) if type(multiList[0]) == int: - multiList = ast.literal_eval("(" + value + ")") + multiList = ast.literal_eval('(' + value + ')') self.__formated_value = multiList @@ -260,7 +259,7 @@ class ParameterConf(): def printattr(self): - print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format) + print 'Parameter[%s]: name = %s, value = %s, format = %s' %(self.id, self.name, self.value, self.format) class OperationConf(): @@ -368,7 +367,9 @@ class OperationConf(): self.parmConfObjList = [] def addParameter(self, name, value, format='str'): - + + if value is None: + return None id = self.__getNewId() parmConfObj = ParameterConf() @@ -428,7 +429,7 @@ class OperationConf(): def printattr(self): - print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME, + print '%s[%s]: name = %s, type = %s, priority = %s' %(self.ELEMENTNAME, self.id, self.name, self.type, @@ -441,12 +442,11 @@ class OperationConf(): if self.type == 'self': - raise ValueError, "This operation type cannot be created" + raise ValueError, 'This operation type cannot be created' - if self.type == 'plotter': - #Plotter(plotter_name) + if self.type == 'plotter': if not plotter_queue: - raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)" + raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)' opObj = Plotter(self.name, plotter_queue) @@ -561,7 +561,7 @@ class ProcUnitConf(): #Compatible with old signal chain version if datatype==None and name==None: - raise ValueError, "datatype or name should be defined" + raise ValueError, 'datatype or name should be defined' if name==None: if 'Proc' in datatype: @@ -592,7 +592,7 @@ class ProcUnitConf(): def addParameter(self, **kwargs): ''' - Add parameters to "run" operation + Add parameters to 'run' operation ''' opObj = self.opConfObjList[0] @@ -630,11 +630,11 @@ class ProcUnitConf(): self.datatype = upElement.get('datatype') self.inputId = upElement.get('inputId') - if self.ELEMENTNAME == "ReadUnit": - self.datatype = self.datatype.replace("Reader", "") + if self.ELEMENTNAME == 'ReadUnit': + self.datatype = self.datatype.replace('Reader', '') - if self.ELEMENTNAME == "ProcUnit": - self.datatype = self.datatype.replace("Proc", "") + if self.ELEMENTNAME == 'ProcUnit': + self.datatype = self.datatype.replace('Proc', '') if self.inputId == 'None': self.inputId = '0' @@ -650,7 +650,7 @@ class ProcUnitConf(): def printattr(self): - print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME, + print '%s[%s]: name = %s, datatype = %s, inputId = %s' %(self.ELEMENTNAME, self.id, self.name, self.datatype, @@ -751,12 +751,12 @@ class ReadUnitConf(ProcUnitConf): return self.ELEMENTNAME - def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="", - endTime="", parentId=None, queue=None, server=None, **kwargs): + def setup(self, id, name, datatype, path='', startDate='', endDate='', + startTime='', endTime='', parentId=None, server=None, **kwargs): #Compatible with old signal chain version if datatype==None and name==None: - raise ValueError, "datatype or name should be defined" + raise ValueError, 'datatype or name should be defined' if name==None: if 'Reader' in datatype: @@ -775,39 +775,28 @@ class ReadUnitConf(ProcUnitConf): self.endDate = endDate self.startTime = startTime self.endTime = endTime - self.inputId = '0' self.parentId = parentId - self.queue = queue self.server = server self.addRunOperation(**kwargs) - def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs): + def update(self, **kwargs): - #Compatible with old signal chain version - if datatype==None and name==None: - raise ValueError, "datatype or name should be defined" - - if name==None: + if 'datatype' in kwargs: + datatype = kwargs.pop('datatype') if 'Reader' in datatype: - name = datatype + self.name = datatype else: - name = '%sReader' %(datatype) - - if datatype==None: - datatype = name.replace('Reader','') - - self.datatype = datatype - self.name = name - self.path = path - self.startDate = startDate - self.endDate = endDate - self.startTime = startTime - self.endTime = endTime + self.name = '%sReader' %(datatype) + self.datatype = self.name.replace('Reader', '') + attrs = ('path', 'startDate', 'endDate', 'startTime', 'endTime', 'parentId') + + for attr in attrs: + if attr in kwargs: + setattr(self, attr, kwargs.pop(attr)) + self.inputId = '0' - self.parentId = parentId - self.updateRunOperation(**kwargs) def removeOperations(self): @@ -822,13 +811,13 @@ class ReadUnitConf(ProcUnitConf): opObj = self.addOperation(name = 'run', optype = 'self') if self.server is None: - opObj.addParameter(name='datatype' , value=self.datatype, format='str') - opObj.addParameter(name='path' , value=self.path, format='str') - opObj.addParameter(name='startDate' , value=self.startDate, format='date') - opObj.addParameter(name='endDate' , value=self.endDate, format='date') - opObj.addParameter(name='startTime' , value=self.startTime, format='time') - opObj.addParameter(name='endTime' , value=self.endTime, format='time') - opObj.addParameter(name='queue' , value=self.queue, format='obj') + opObj.addParameter(name='datatype', value=self.datatype, format='str') + opObj.addParameter(name='path', value=self.path, format='str') + opObj.addParameter(name='startDate', value=self.startDate, format='date') + opObj.addParameter(name='endDate', value=self.endDate, format='date') + opObj.addParameter(name='startTime', value=self.startTime, format='time') + opObj.addParameter(name='endTime', value=self.endTime, format='time') + for key, value in kwargs.items(): opObj.addParameter(name=key, value=value, format=type(value).__name__) else: @@ -839,32 +828,21 @@ class ReadUnitConf(ProcUnitConf): def updateRunOperation(self, **kwargs): - opObj = self.getOperationObj(name = 'run') + opObj = self.getOperationObj(name='run') opObj.removeParameters() - opObj.addParameter(name='datatype' , value=self.datatype, format='str') - opObj.addParameter(name='path' , value=self.path, format='str') - opObj.addParameter(name='startDate' , value=self.startDate, format='date') - opObj.addParameter(name='endDate' , value=self.endDate, format='date') - opObj.addParameter(name='startTime' , value=self.startTime, format='time') - opObj.addParameter(name='endTime' , value=self.endTime, format='time') - + opObj.addParameter(name='datatype', value=self.datatype, format='str') + opObj.addParameter(name='path', value=self.path, format='str') + opObj.addParameter(name='startDate', value=self.startDate, format='date') + opObj.addParameter(name='endDate', value=self.endDate, format='date') + opObj.addParameter(name='startTime', value=self.startTime, format='time') + opObj.addParameter(name='endTime', value=self.endTime, format='time') + for key, value in kwargs.items(): opObj.addParameter(name=key, value=value, format=type(value).__name__) return opObj - # def makeXml(self, projectElement): - # - # procUnitElement = SubElement(projectElement, self.ELEMENTNAME) - # procUnitElement.set('id', str(self.id)) - # procUnitElement.set('name', self.name) - # procUnitElement.set('datatype', self.datatype) - # procUnitElement.set('inputId', str(self.inputId)) - # - # for opConfObj in self.opConfObjList: - # opConfObj.makeXml(procUnitElement) - def readXml(self, upElement): self.id = upElement.get('id') @@ -872,8 +850,8 @@ class ReadUnitConf(ProcUnitConf): self.datatype = upElement.get('datatype') self.inputId = upElement.get('inputId') - if self.ELEMENTNAME == "ReadUnit": - self.datatype = self.datatype.replace("Reader", "") + if self.ELEMENTNAME == 'ReadUnit': + self.datatype = self.datatype.replace('Reader', '') if self.inputId == 'None': self.inputId = '0' @@ -894,10 +872,10 @@ class ReadUnitConf(ProcUnitConf): self.startTime = opConfObj.getParameterValue('startTime') self.endTime = opConfObj.getParameterValue('endTime') -class Project(): +class Project(Process): id = None - name = None + # name = None description = None filename = None @@ -909,8 +887,9 @@ class Project(): def __init__(self, plotter_queue=None): + Process.__init__(self) self.id = None - self.name = None + # self.name = None self.description = None self.plotterQueue = plotter_queue @@ -962,17 +941,26 @@ class Project(): self.procUnitConfObjDict = newProcUnitConfObjDict - def setup(self, id, name, description): + def setup(self, id, name='', description=''): + print + print '*'*60 + print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__ + print '*'*60 + print self.id = str(id) - self.name = name self.description = description def update(self, name, description): - self.name = name self.description = description + def clone(self): + + p = Project() + p.procUnitConfObjDict = self.procUnitConfObjDict + return p + def addReadUnit(self, id=None, datatype=None, name=None, **kwargs): if id is None: @@ -1012,7 +1000,7 @@ class Project(): def getReadUnitObj(self): for obj in self.procUnitConfObjDict.values(): - if obj.getElementName() == "ReadUnit": + if obj.getElementName() == 'ReadUnit': return obj return None @@ -1057,20 +1045,20 @@ class Project(): if self.filename: filename = self.filename else: - filename = "schain.xml" + filename = 'schain.xml' if not filename: - print "filename has not been defined. Use setFilename(filename) for do it." + print 'filename has not been defined. Use setFilename(filename) for do it.' return 0 abs_file = os.path.abspath(filename) if not os.access(os.path.dirname(abs_file), os.W_OK): - print "No write permission on %s" %os.path.dirname(abs_file) + print 'No write permission on %s' %os.path.dirname(abs_file) return 0 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)): - print "File %s already exists and it could not be overwriten" %abs_file + print 'File %s already exists and it could not be overwriten' %abs_file return 0 self.makeXml() @@ -1084,13 +1072,13 @@ class Project(): def readXml(self, filename = None): if not filename: - print "filename is not defined" + print 'filename is not defined' return 0 abs_file = os.path.abspath(filename) if not os.path.isfile(abs_file): - print "%s file does not exist" %abs_file + print '%s file does not exist' %abs_file return 0 self.projectElement = None @@ -1099,7 +1087,7 @@ class Project(): try: self.projectElement = ElementTree().parse(abs_file) except: - print "Error reading %s, verify file format" %filename + print 'Error reading %s, verify file format' %filename return 0 self.project = self.projectElement.tag @@ -1136,7 +1124,7 @@ class Project(): def printattr(self): - print "Project[%s]: name = %s, description = %s" %(self.id, + print 'Project[%s]: name = %s, description = %s' %(self.id, self.name, self.description) @@ -1170,7 +1158,7 @@ class Project(): self.__connect(puObjIN, thisPUObj) - def __handleError(self, procUnitConfObj, send_email=True): + def __handleError(self, procUnitConfObj, send_email=False): import socket @@ -1178,33 +1166,33 @@ class Project(): sys.exc_info()[1], sys.exc_info()[2]) - print "***** Error occurred in %s *****" %(procUnitConfObj.name) - print "***** %s" %err[-1] + print '***** Error occurred in %s *****' %(procUnitConfObj.name) + print '***** %s' %err[-1] - message = "".join(err) + message = ''.join(err) sys.stderr.write(message) if not send_email: return - subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name) + subject = 'SChain v%s: Error running %s\n' %(schainpy.__version__, procUnitConfObj.name) - subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name) - subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname()) - subtitle += "Working directory: %s\n" %os.path.abspath("./") - subtitle += "Configuration file: %s\n" %self.filename - subtitle += "Time: %s\n" %str(datetime.datetime.now()) + subtitle = '%s: %s\n' %(procUnitConfObj.getElementName() ,procUnitConfObj.name) + subtitle += 'Hostname: %s\n' %socket.gethostbyname(socket.gethostname()) + subtitle += 'Working directory: %s\n' %os.path.abspath('./') + subtitle += 'Configuration file: %s\n' %self.filename + subtitle += 'Time: %s\n' %str(datetime.datetime.now()) readUnitConfObj = self.getReadUnitObj() if readUnitConfObj: - subtitle += "\nInput parameters:\n" - subtitle += "[Data path = %s]\n" %readUnitConfObj.path - subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype - subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate - subtitle += "[End date = %s]\n" %readUnitConfObj.endDate - subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime - subtitle += "[End time = %s]\n" %readUnitConfObj.endTime + subtitle += '\nInput parameters:\n' + subtitle += '[Data path = %s]\n' %readUnitConfObj.path + subtitle += '[Data type = %s]\n' %readUnitConfObj.datatype + subtitle += '[Start date = %s]\n' %readUnitConfObj.startDate + subtitle += '[End date = %s]\n' %readUnitConfObj.endDate + subtitle += '[Start time = %s]\n' %readUnitConfObj.startTime + subtitle += '[End time = %s]\n' %readUnitConfObj.endTime adminObj = schainpy.admin.SchainNotify() adminObj.sendAlert(message=message, @@ -1219,15 +1207,15 @@ class Project(): return 0 def runController(self): - """ + ''' returns 0 when this process has been stopped, 1 otherwise - """ + ''' if self.isPaused(): - print "Process suspended" + print 'Process suspended' while True: - sleep(0.1) + time.sleep(0.1) if not self.isPaused(): break @@ -1235,10 +1223,10 @@ class Project(): if self.isStopped(): break - print "Process reinitialized" + print 'Process reinitialized' if self.isStopped(): - print "Process stopped" + print 'Process stopped' return 0 return 1 @@ -1249,23 +1237,22 @@ class Project(): def setPlotterQueue(self, plotter_queue): - raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class" + raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class' def getPlotterQueue(self): - raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class" + raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class' def useExternalPlotter(self): - raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class" + raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class' def run(self): - print - print "*"*60 - print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__ - print "*"*60 - print + log.success('Starting {}'.format(self.name)) + + self.createObjects() + self.connectObjects() keyList = self.procUnitConfObjDict.keys() keyList.sort() @@ -1275,7 +1262,6 @@ class Project(): is_ok = False for procKey in keyList: -# print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id) procUnitConfObj = self.procUnitConfObjDict[procKey] @@ -1286,19 +1272,18 @@ class Project(): is_ok = False break except ValueError, e: - sleep(0.5) + time.sleep(0.5) self.__handleError(procUnitConfObj, send_email=True) is_ok = False break except: - sleep(0.5) + time.sleep(0.5) self.__handleError(procUnitConfObj) is_ok = False break #If every process unit finished so end process if not(is_ok): -# print "Every process unit have finished" break if not self.runController(): @@ -1309,11 +1294,4 @@ class Project(): procUnitConfObj = self.procUnitConfObjDict[procKey] procUnitConfObj.close() - print "Process finished" - - def start(self, filename=None): - - self.writeXml(filename) - self.createObjects() - self.connectObjects() - self.run() + log.success('{} finished'.format(self.name)) diff --git a/schainpy/controller_api.py b/schainpy/controller_api.py index 4e48082..2d1775e 100644 --- a/schainpy/controller_api.py +++ b/schainpy/controller_api.py @@ -14,8 +14,8 @@ class ControllerThread(threading.Thread, Project): self.setDaemon(True) self.lock = threading.Lock() - self.control = {'stop':False, 'pause':False} - + self.control = { 'stop':False, 'pause':False } + def __del__(self): self.control['stop'] = True diff --git a/schainpy/gui/viewcontroller/basicwindow.py b/schainpy/gui/viewcontroller/basicwindow.py index caf1333..c2d8e75 100644 --- a/schainpy/gui/viewcontroller/basicwindow.py +++ b/schainpy/gui/viewcontroller/basicwindow.py @@ -6218,3 +6218,6 @@ class ShowMeConsole(QtCore.QObject): text = text[:-1] self.textWritten.emit(str(text)) + + def flush(self): + pass diff --git a/schainpy/model/__init__.py b/schainpy/model/__init__.py index 6dea354..3d76a6d 100644 --- a/schainpy/model/__init__.py +++ b/schainpy/model/__init__.py @@ -9,4 +9,4 @@ from data import * from io import * from proc import * from graphics import * -from utils import * +from utils import * \ No newline at end of file diff --git a/schainpy/model/data/jrodata.py b/schainpy/model/data/jrodata.py index 73bf497..bed4822 100644 --- a/schainpy/model/data/jrodata.py +++ b/schainpy/model/data/jrodata.py @@ -700,7 +700,7 @@ class Spectra(JROData): for pair in pairsList: if pair not in self.pairsList: raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair) - pairsIndexList.append(self.pairsList.index(pair)) + pairsIndexList.append(self.pairsList.index(pair)) for i in range(len(pairsIndexList)): pair = self.pairsList[pairsIndexList[i]] ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0) diff --git a/schainpy/model/graphics/figure.py b/schainpy/model/graphics/figure.py index c31e9b3..85fd5bd 100644 --- a/schainpy/model/graphics/figure.py +++ b/schainpy/model/graphics/figure.py @@ -61,7 +61,7 @@ class Figure(Operation): figfile = None created = False - + parameters = {} def __init__(self, **kwargs): Operation.__init__(self, **kwargs) diff --git a/schainpy/model/graphics/jroplot_correlation.py b/schainpy/model/graphics/jroplot_correlation.py index 9f12d54..474b860 100644 --- a/schainpy/model/graphics/jroplot_correlation.py +++ b/schainpy/model/graphics/jroplot_correlation.py @@ -2,11 +2,10 @@ import os import datetime import numpy import copy - +from schainpy.model import * from figure import Figure, isRealtime class CorrelationPlot(Figure): - isConfig = None __nsubplots = None diff --git a/schainpy/model/graphics/jroplot_data.py b/schainpy/model/graphics/jroplot_data.py index 720428c..8316adf 100644 --- a/schainpy/model/graphics/jroplot_data.py +++ b/schainpy/model/graphics/jroplot_data.py @@ -1,30 +1,33 @@ import os -import zmq import time -import numpy +import glob import datetime -import numpy as np +from multiprocessing import Process + +import zmq +import numpy import matplotlib -import glob -matplotlib.use('TkAgg') import matplotlib.pyplot as plt from mpl_toolkits.axes_grid1 import make_axes_locatable -from matplotlib.ticker import FuncFormatter, LinearLocator -from multiprocessing import Process +from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator from schainpy.model.proc.jroproc_base import Operation - -plt.ion() +from schainpy.utils import log func = lambda x, pos: ('%s') %(datetime.datetime.fromtimestamp(x).strftime('%H:%M')) -d1970 = datetime.datetime(1970,1,1) +d1970 = datetime.datetime(1970, 1, 1) + class PlotData(Operation, Process): + ''' + Base class for Schain plotting operations + ''' CODE = 'Figure' colormap = 'jro' + bgcolor = 'white' CONFLATE = False __MAXNUMX = 80 __missing = 1E30 @@ -35,54 +38,143 @@ class PlotData(Operation, Process): Process.__init__(self) self.kwargs['code'] = self.CODE self.mp = False - self.dataOut = None - self.isConfig = False - self.figure = None + self.data = None + self.isConfig = False + self.figures = [] self.axes = [] + self.cb_axes = [] self.localtime = kwargs.pop('localtime', True) self.show = kwargs.get('show', True) self.save = kwargs.get('save', False) self.colormap = kwargs.get('colormap', self.colormap) self.colormap_coh = kwargs.get('colormap_coh', 'jet') self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r') - self.showprofile = kwargs.get('showprofile', True) - self.title = kwargs.get('wintitle', '') + self.colormaps = kwargs.get('colormaps', None) + self.bgcolor = kwargs.get('bgcolor', self.bgcolor) + self.showprofile = kwargs.get('showprofile', False) + self.title = kwargs.get('wintitle', self.CODE.upper()) + self.cb_label = kwargs.get('cb_label', None) + self.cb_labels = kwargs.get('cb_labels', None) self.xaxis = kwargs.get('xaxis', 'frequency') self.zmin = kwargs.get('zmin', None) self.zmax = kwargs.get('zmax', None) + self.zlimits = kwargs.get('zlimits', None) self.xmin = kwargs.get('xmin', None) + if self.xmin is not None: + self.xmin += 5 self.xmax = kwargs.get('xmax', None) self.xrange = kwargs.get('xrange', 24) self.ymin = kwargs.get('ymin', None) self.ymax = kwargs.get('ymax', None) - self.__MAXNUMY = kwargs.get('decimation', 80) - self.throttle_value = 5 - self.times = [] - #self.interactive = self.kwargs['parent'] + self.xlabel = kwargs.get('xlabel', None) + self.__MAXNUMY = kwargs.get('decimation', 100) + self.showSNR = kwargs.get('showSNR', False) + self.oneFigure = kwargs.get('oneFigure', True) + self.width = kwargs.get('width', None) + self.height = kwargs.get('height', None) + self.colorbar = kwargs.get('colorbar', True) + self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1]) + self.titles = ['' for __ in range(16)] + + def __setup(self): + ''' + Common setup for all figures, here figures and axes are created + ''' + + self.setup() + if self.width is None: + self.width = 8 + + self.figures = [] + self.axes = [] + self.cb_axes = [] + self.pf_axes = [] + self.cmaps = [] + + size = '15%' if self.ncols==1 else '30%' + pad = '4%' if self.ncols==1 else '8%' + + if self.oneFigure: + if self.height is None: + self.height = 1.4*self.nrows + 1 + fig = plt.figure(figsize=(self.width, self.height), + edgecolor='k', + facecolor='w') + self.figures.append(fig) + for n in range(self.nplots): + ax = fig.add_subplot(self.nrows, self.ncols, n+1) + ax.tick_params(labelsize=8) + ax.firsttime = True + self.axes.append(ax) + if self.showprofile: + cax = self.__add_axes(ax, size=size, pad=pad) + cax.tick_params(labelsize=8) + self.pf_axes.append(cax) + else: + if self.height is None: + self.height = 3 + for n in range(self.nplots): + fig = plt.figure(figsize=(self.width, self.height), + edgecolor='k', + facecolor='w') + ax = fig.add_subplot(1, 1, 1) + ax.tick_params(labelsize=8) + ax.firsttime = True + self.figures.append(fig) + self.axes.append(ax) + if self.showprofile: + cax = self.__add_axes(ax, size=size, pad=pad) + cax.tick_params(labelsize=8) + self.pf_axes.append(cax) + + for n in range(self.nrows): + if self.colormaps is not None: + cmap = plt.get_cmap(self.colormaps[n]) + else: + cmap = plt.get_cmap(self.colormap) + cmap.set_bad(self.bgcolor, 1.) + self.cmaps.append(cmap) + + def __add_axes(self, ax, size='30%', pad='8%'): ''' - this new parameter is created to plot data from varius channels at different figures - 1. crear una lista de figuras donde se puedan plotear las figuras, - 2. dar las opciones de configuracion a cada figura, estas opciones son iguales para ambas figuras - 3. probar? + Add new axes to the given figure ''' - self.ind_plt_ch = kwargs.get('ind_plt_ch', False) - self.figurelist = None + divider = make_axes_locatable(ax) + nax = divider.new_horizontal(size=size, pad=pad) + ax.figure.add_axes(nax) + return nax - def fill_gaps(self, x_buffer, y_buffer, z_buffer): + def setup(self): + ''' + This method should be implemented in the child class, the following + attributes should be set: + + self.nrows: number of rows + self.ncols: number of cols + self.nplots: number of plots (channels or pairs) + self.ylabel: label for Y axes + self.titles: list of axes title + + ''' + raise(NotImplementedError, 'Implement this method in child class') + def fill_gaps(self, x_buffer, y_buffer, z_buffer): + ''' + Create a masked array for missing data + ''' if x_buffer.shape[0] < 2: return x_buffer, y_buffer, z_buffer deltas = x_buffer[1:] - x_buffer[0:-1] - x_median = np.median(deltas) + x_median = numpy.median(deltas) - index = np.where(deltas > 5*x_median) + index = numpy.where(deltas > 5*x_median) if len(index[0]) != 0: z_buffer[::, index[0], ::] = self.__missing - z_buffer = np.ma.masked_inside(z_buffer, + z_buffer = numpy.ma.masked_inside(z_buffer, 0.99*self.__missing, 1.01*self.__missing) @@ -97,110 +189,117 @@ class PlotData(Operation, Process): x = self.x y = self.y[::dy] z = self.z[::, ::, ::dy] - + return x, y, z - ''' - JM: - elimana las otras imagenes generadas debido a que lso workers no llegan en orden y le pueden - poner otro tiempo a la figura q no necesariamente es el ultimo. - Solo se realiza cuando termina la imagen. - Problemas: + def format(self): + ''' + Set min and max values, labels, ticks and titles + ''' - File "/home/ci-81/workspace/schainv2.3/schainpy/model/graphics/jroplot_data.py", line 145, in __plot - for n, eachfigure in enumerate(self.figurelist): - TypeError: 'NoneType' object is not iterable + if self.xmin is None: + xmin = self.min_time + else: + if self.xaxis is 'time': + dt = datetime.datetime.fromtimestamp(self.min_time) + xmin = (datetime.datetime.combine(dt.date(), + datetime.time(int(self.xmin), 0, 0))-d1970).total_seconds() + else: + xmin = self.xmin - ''' - def deleteanotherfiles(self): - figurenames=[] - if self.figurelist != None: - for n, eachfigure in enumerate(self.figurelist): - #add specific name for each channel in channelList - ghostfigname = os.path.join(self.save, '{}_{}_{}'.format(self.titles[n].replace(' ',''),self.CODE, - datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d'))) - figname = os.path.join(self.save, '{}_{}_{}.png'.format(self.titles[n].replace(' ',''),self.CODE, - datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S'))) - - for ghostfigure in glob.glob(ghostfigname+'*'): #ghostfigure will adopt all posible names of figures - if ghostfigure != figname: - os.remove(ghostfigure) - print 'Removing GhostFigures:' , figname - else : - '''Erasing ghost images for just on******************''' - ghostfigname = os.path.join(self.save, '{}_{}'.format(self.CODE,datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d'))) - figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S'))) - for ghostfigure in glob.glob(ghostfigname+'*'): #ghostfigure will adopt all posible names of figures - if ghostfigure != figname: - os.remove(ghostfigure) - print 'Removing GhostFigures:' , figname + if self.xmax is None: + xmax = xmin+self.xrange*60*60 + else: + if self.xaxis is 'time': + dt = datetime.datetime.fromtimestamp(self.min_time) + xmax = (datetime.datetime.combine(dt.date(), + datetime.time(int(self.xmax), 0, 0))-d1970).total_seconds() + else: + xmax = self.xmax + + ymin = self.ymin if self.ymin else numpy.nanmin(self.y) + ymax = self.ymax if self.ymax else numpy.nanmax(self.y) + + ystep = 200 if ymax>= 800 else 100 if ymax>=400 else 50 if ymax>=200 else 20 + + for n, ax in enumerate(self.axes): + if ax.firsttime: + ax.set_facecolor(self.bgcolor) + ax.yaxis.set_major_locator(MultipleLocator(ystep)) + if self.xaxis is 'time': + ax.xaxis.set_major_formatter(FuncFormatter(func)) + ax.xaxis.set_major_locator(LinearLocator(9)) + if self.xlabel is not None: + ax.set_xlabel(self.xlabel) + ax.set_ylabel(self.ylabel) + ax.firsttime = False + if self.showprofile: + self.pf_axes[n].set_ylim(ymin, ymax) + self.pf_axes[n].set_xlim(self.zmin, self.zmax) + self.pf_axes[n].set_xlabel('dB') + self.pf_axes[n].grid(b=True, axis='x') + [tick.set_visible(False) for tick in self.pf_axes[n].get_yticklabels()] + if self.colorbar: + cb = plt.colorbar(ax.plt, ax=ax, pad=0.02) + cb.ax.tick_params(labelsize=8) + if self.cb_label: + cb.set_label(self.cb_label, size=8) + elif self.cb_labels: + cb.set_label(self.cb_labels[n], size=8) + + ax.set_title('{} - {} UTC'.format( + self.titles[n], + datetime.datetime.fromtimestamp(self.max_time).strftime('%H:%M:%S')), + size=8) + ax.set_xlim(xmin, xmax) + ax.set_ylim(ymin, ymax) + def __plot(self): - - print 'plotting...{}'.format(self.CODE) - if self.ind_plt_ch is False : #standard + ''' + ''' + log.success('Plotting', self.name) + + self.plot() + self.format() + + for n, fig in enumerate(self.figures): + if self.nrows == 0 or self.nplots == 0: + log.warning('No data', self.name) + continue if self.show: - self.figure.show() - self.plot() - plt.tight_layout() - self.figure.canvas.manager.set_window_title('{} {} - {}'.format(self.title, self.CODE.upper(), - datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d'))) - else : - print 'len(self.figurelist): ',len(self.figurelist) - for n, eachfigure in enumerate(self.figurelist): - if self.show: - eachfigure.show() - - self.plot() - eachfigure.tight_layout() # ajuste de cada subplot - eachfigure.canvas.manager.set_window_title('{} {} - {}'.format(self.title[n], self.CODE.upper(), - datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d'))) - - # if self.save: - # if self.ind_plt_ch is False : #standard - # figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE, - # datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S'))) - # print 'Saving figure: {}'.format(figname) - # self.figure.savefig(figname) - # else : - # for n, eachfigure in enumerate(self.figurelist): - # #add specific name for each channel in channelList - # figname = os.path.join(self.save, '{}_{}_{}.png'.format(self.titles[n],self.CODE, - # datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S'))) - # - # print 'Saving figure: {}'.format(figname) - # eachfigure.savefig(figname) - - if self.ind_plt_ch is False : - self.figure.canvas.draw() - else : - for eachfigure in self.figurelist: - eachfigure.canvas.draw() - - if self.save: - if self.ind_plt_ch is False : #standard - figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE, - datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S'))) + fig.show() + + fig.tight_layout() + fig.canvas.manager.set_window_title('{} - {}'.format(self.title, + datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d'))) + # fig.canvas.draw() + + if self.save and self.data.ended: + channels = range(self.nrows) + if self.oneFigure: + label = '' + else: + label = '_{}'.format(channels[n]) + figname = os.path.join( + self.save, + '{}{}_{}.png'.format( + self.CODE, + label, + datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S') + ) + ) print 'Saving figure: {}'.format(figname) - self.figure.savefig(figname) - else : - for n, eachfigure in enumerate(self.figurelist): - #add specific name for each channel in channelList - figname = os.path.join(self.save, '{}_{}_{}.png'.format(self.titles[n].replace(' ',''),self.CODE, - datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S'))) - - print 'Saving figure: {}'.format(figname) - eachfigure.savefig(figname) - + fig.savefig(figname) def plot(self): - - print 'plotting...{}'.format(self.CODE.upper()) - return + ''' + ''' + raise(NotImplementedError, 'Implement this method in child class') def run(self): - print '[Starting] {}'.format(self.name) + log.success('Starting', self.name) context = zmq.Context() receiver = context.socket(zmq.SUB) @@ -210,152 +309,104 @@ class PlotData(Operation, Process): if 'server' in self.kwargs['parent']: receiver.connect('ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server'])) else: - receiver.connect("ipc:///tmp/zmq.plots") - - seconds_passed = 0 + receiver.connect("ipc:///tmp/zmq.plots") while True: try: - self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)#flags=zmq.NOBLOCK - self.started = self.data['STARTED'] - self.dataOut = self.data['dataOut'] - - if (len(self.times) < len(self.data['times']) and not self.started and self.data['ENDED']): - continue - - self.times = self.data['times'] - self.times.sort() - self.throttle_value = self.data['throttle'] - self.min_time = self.times[0] - self.max_time = self.times[-1] + self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK) + + self.min_time = self.data.times[0] + self.max_time = self.data.times[-1] if self.isConfig is False: - print 'setting up' - self.setup() + self.__setup() self.isConfig = True - self.__plot() - - if self.data['ENDED'] is True: - print '********GRAPHIC ENDED********' - self.ended = True - self.isConfig = False - self.__plot() - self.deleteanotherfiles() #CLPDG - elif seconds_passed >= self.data['throttle']: - print 'passed', seconds_passed - self.__plot() - seconds_passed = 0 + + self.__plot() except zmq.Again as e: - print 'Waiting for data...' - plt.pause(2) - seconds_passed += 2 + log.log('Waiting for data...') + if self.data: + plt.pause(self.data.throttle) + else: + time.sleep(2) def close(self): - if self.dataOut: + if self.data: self.__plot() class PlotSpectraData(PlotData): + ''' + Plot for Spectra data + ''' CODE = 'spc' - colormap = 'jro' - CONFLATE = False + colormap = 'jro' def setup(self): - - ncolspan = 1 - colspan = 1 - self.ncols = int(numpy.sqrt(self.dataOut.nChannels)+0.9) - self.nrows = int(self.dataOut.nChannels*1./self.ncols + 0.9) - self.width = 3.6*self.ncols - self.height = 3.2*self.nrows - if self.showprofile: - ncolspan = 3 - colspan = 2 - self.width += 1.2*self.ncols + self.nplots = len(self.data.channels) + self.ncols = int(numpy.sqrt(self.nplots)+ 0.9) + self.nrows = int((1.0*self.nplots/self.ncols) + 0.9) + self.width = 3.4*self.ncols + self.height = 3*self.nrows + self.cb_label = 'dB' + if self.showprofile: + self.width += 0.8*self.ncols self.ylabel = 'Range [Km]' - self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList] - - if self.figure is None: - self.figure = plt.figure(figsize=(self.width, self.height), - edgecolor='k', - facecolor='w') - else: - self.figure.clf() - - n = 0 - for y in range(self.nrows): - for x in range(self.ncols): - if n >= self.dataOut.nChannels: - break - ax = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan), 1, colspan) - if self.showprofile: - ax.ax_profile = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan+colspan), 1, 1) - - ax.firsttime = True - self.axes.append(ax) - n += 1 def plot(self): - if self.xaxis == "frequency": - x = self.dataOut.getFreqRange(1)/1000. - xlabel = "Frequency (kHz)" + x = self.data.xrange[0] + self.xlabel = "Frequency (kHz)" elif self.xaxis == "time": - x = self.dataOut.getAcfRange(1) - xlabel = "Time (ms)" + x = self.data.xrange[1] + self.xlabel = "Time (ms)" else: - x = self.dataOut.getVelRange(1) - xlabel = "Velocity (m/s)" + x = self.data.xrange[2] + self.xlabel = "Velocity (m/s)" - y = self.dataOut.getHeiRange() - z = self.data[self.CODE] + if self.CODE == 'spc_mean': + x = self.data.xrange[2] + self.xlabel = "Velocity (m/s)" + self.titles = [] + + y = self.data.heights + self.y = y + z = self.data['spc'] + for n, ax in enumerate(self.axes): + noise = self.data['noise'][n][-1] + if self.CODE == 'spc_mean': + mean = self.data['mean'][n][-1] if ax.firsttime: - self.xmax = self.xmax if self.xmax else np.nanmax(x) + self.xmax = self.xmax if self.xmax else numpy.nanmax(x) self.xmin = self.xmin if self.xmin else -self.xmax - self.ymin = self.ymin if self.ymin else np.nanmin(y) - self.ymax = self.ymax if self.ymax else np.nanmax(y) - self.zmin = self.zmin if self.zmin else np.nanmin(z) - self.zmax = self.zmax if self.zmax else np.nanmax(z) - ax.plot = ax.pcolormesh(x, y, z[n].T, - vmin=self.zmin, - vmax=self.zmax, - cmap=plt.get_cmap(self.colormap) - ) - divider = make_axes_locatable(ax) - cax = divider.new_horizontal(size='3%', pad=0.05) - self.figure.add_axes(cax) - plt.colorbar(ax.plot, cax) - - ax.set_xlim(self.xmin, self.xmax) - ax.set_ylim(self.ymin, self.ymax) - - ax.set_ylabel(self.ylabel) - ax.set_xlabel(xlabel) - - ax.firsttime = False + self.zmin = self.zmin if self.zmin else numpy.nanmin(z) + self.zmax = self.zmax if self.zmax else numpy.nanmax(z) + ax.plt = ax.pcolormesh(x, y, z[n].T, + vmin=self.zmin, + vmax=self.zmax, + cmap=plt.get_cmap(self.colormap) + ) if self.showprofile: - ax.plot_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0] - ax.ax_profile.set_xlim(self.zmin, self.zmax) - ax.ax_profile.set_ylim(self.ymin, self.ymax) - ax.ax_profile.set_xlabel('dB') - ax.ax_profile.grid(b=True, axis='x') - ax.plot_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y, - color="k", linestyle="dashed", lw=2)[0] - [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()] + ax.plt_profile= self.pf_axes[n].plot(self.data['rti'][n][-1], y)[0] + ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y, + color="k", linestyle="dashed", lw=1)[0] + if self.CODE == 'spc_mean': + ax.plt_mean = ax.plot(mean, y, color='k')[0] else: - ax.plot.set_array(z[n].T.ravel()) + ax.plt.set_array(z[n].T.ravel()) if self.showprofile: - ax.plot_profile.set_data(self.data['rti'][self.max_time][n], y) - ax.plot_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y) + ax.plt_profile.set_data(self.data['rti'][n][-1], y) + ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y) + if self.CODE == 'spc_mean': + ax.plt_mean.set_data(mean, y) - ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]), - size=8) + self.titles.append('CH {}: {:3.2f}dB'.format(n, noise)) self.saveTime = self.max_time @@ -365,538 +416,245 @@ class PlotCrossSpectraData(PlotData): zmin_coh = None zmax_coh = None zmin_phase = None - zmax_phase = None - CONFLATE = False + zmax_phase = None def setup(self): - ncolspan = 1 - colspan = 1 - self.ncols = 2 - self.nrows = self.dataOut.nPairs - self.width = 3.6*self.ncols - self.height = 3.2*self.nrows - + self.ncols = 4 + self.nrows = len(self.data.pairs) + self.nplots = self.nrows*4 + self.width = 3.4*self.ncols + self.height = 3*self.nrows self.ylabel = 'Range [Km]' - self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList] - - if self.figure is None: - self.figure = plt.figure(figsize=(self.width, self.height), - edgecolor='k', - facecolor='w') - else: - self.figure.clf() - - for y in range(self.nrows): - for x in range(self.ncols): - ax = plt.subplot2grid((self.nrows, self.ncols), (y, x), 1, 1) - ax.firsttime = True - self.axes.append(ax) + self.showprofile = False def plot(self): if self.xaxis == "frequency": - x = self.dataOut.getFreqRange(1)/1000. - xlabel = "Frequency (kHz)" + x = self.data.xrange[0] + self.xlabel = "Frequency (kHz)" elif self.xaxis == "time": - x = self.dataOut.getAcfRange(1) - xlabel = "Time (ms)" + x = self.data.xrange[1] + self.xlabel = "Time (ms)" else: - x = self.dataOut.getVelRange(1) - xlabel = "Velocity (m/s)" + x = self.data.xrange[2] + self.xlabel = "Velocity (m/s)" - y = self.dataOut.getHeiRange() - z_coh = self.data['cspc_coh'] - z_phase = self.data['cspc_phase'] + self.titles = [] + + y = self.data.heights + self.y = y + spc = self.data['spc'] + cspc = self.data['cspc'] for n in range(self.nrows): - ax = self.axes[2*n] - ax1 = self.axes[2*n+1] + noise = self.data['noise'][n][-1] + pair = self.data.pairs[n] + ax = self.axes[4*n] + ax3 = self.axes[4*n+3] if ax.firsttime: - self.xmax = self.xmax if self.xmax else np.nanmax(x) + self.xmax = self.xmax if self.xmax else numpy.nanmax(x) self.xmin = self.xmin if self.xmin else -self.xmax - self.ymin = self.ymin if self.ymin else np.nanmin(y) - self.ymax = self.ymax if self.ymax else np.nanmax(y) - self.zmin_coh = self.zmin_coh if self.zmin_coh else 0.0 - self.zmax_coh = self.zmax_coh if self.zmax_coh else 1.0 - self.zmin_phase = self.zmin_phase if self.zmin_phase else -180 - self.zmax_phase = self.zmax_phase if self.zmax_phase else 180 - - ax.plot = ax.pcolormesh(x, y, z_coh[n].T, - vmin=self.zmin_coh, - vmax=self.zmax_coh, - cmap=plt.get_cmap(self.colormap_coh) - ) - divider = make_axes_locatable(ax) - cax = divider.new_horizontal(size='3%', pad=0.05) - self.figure.add_axes(cax) - plt.colorbar(ax.plot, cax) - - ax.set_xlim(self.xmin, self.xmax) - ax.set_ylim(self.ymin, self.ymax) - - ax.set_ylabel(self.ylabel) - ax.set_xlabel(xlabel) - ax.firsttime = False - - ax1.plot = ax1.pcolormesh(x, y, z_phase[n].T, - vmin=self.zmin_phase, - vmax=self.zmax_phase, - cmap=plt.get_cmap(self.colormap_phase) - ) - divider = make_axes_locatable(ax1) - cax = divider.new_horizontal(size='3%', pad=0.05) - self.figure.add_axes(cax) - plt.colorbar(ax1.plot, cax) - - ax1.set_xlim(self.xmin, self.xmax) - ax1.set_ylim(self.ymin, self.ymax) - - ax1.set_ylabel(self.ylabel) - ax1.set_xlabel(xlabel) - ax1.firsttime = False + self.zmin = self.zmin if self.zmin else numpy.nanmin(spc) + self.zmax = self.zmax if self.zmax else numpy.nanmax(spc) + ax.plt = ax.pcolormesh(x, y, spc[pair[0]].T, + vmin=self.zmin, + vmax=self.zmax, + cmap=plt.get_cmap(self.colormap) + ) else: - ax.plot.set_array(z_coh[n].T.ravel()) - ax1.plot.set_array(z_phase[n].T.ravel()) - - ax.set_title('Coherence Ch{} * Ch{}'.format(self.dataOut.pairsList[n][0], self.dataOut.pairsList[n][1]), size=8) - ax1.set_title('Phase Ch{} * Ch{}'.format(self.dataOut.pairsList[n][0], self.dataOut.pairsList[n][1]), size=8) - self.saveTime = self.max_time - - -class PlotSpectraMeanData(PlotSpectraData): + ax.plt.set_array(spc[pair[0]].T.ravel()) + self.titles.append('CH {}: {:3.2f}dB'.format(n, noise)) - CODE = 'spc_mean' - colormap = 'jet' - - def plot(self): - - if self.xaxis == "frequency": - x = self.dataOut.getFreqRange(1)/1000. - xlabel = "Frequency (kHz)" - elif self.xaxis == "time": - x = self.dataOut.getAcfRange(1) - xlabel = "Time (ms)" - else: - x = self.dataOut.getVelRange(1) - xlabel = "Velocity (m/s)" - - y = self.dataOut.getHeiRange() - z = self.data['spc'] - mean = self.data['mean'][self.max_time] - - for n, ax in enumerate(self.axes): - - if ax.firsttime: - self.xmax = self.xmax if self.xmax else np.nanmax(x) - self.xmin = self.xmin if self.xmin else -self.xmax - self.ymin = self.ymin if self.ymin else np.nanmin(y) - self.ymax = self.ymax if self.ymax else np.nanmax(y) - self.zmin = self.zmin if self.zmin else np.nanmin(z) - self.zmax = self.zmax if self.zmax else np.nanmax(z) - ax.plt = ax.pcolormesh(x, y, z[n].T, + ax = self.axes[4*n+1] + if ax.firsttime: + ax.plt = ax.pcolormesh(x, y, spc[pair[1]].T, vmin=self.zmin, vmax=self.zmax, cmap=plt.get_cmap(self.colormap) ) - ax.plt_dop = ax.plot(mean[n], y, - color='k')[0] - - divider = make_axes_locatable(ax) - cax = divider.new_horizontal(size='3%', pad=0.05) - self.figure.add_axes(cax) - plt.colorbar(ax.plt, cax) - - ax.set_xlim(self.xmin, self.xmax) - ax.set_ylim(self.ymin, self.ymax) - - ax.set_ylabel(self.ylabel) - ax.set_xlabel(xlabel) - - ax.firsttime = False - - if self.showprofile: - ax.plt_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0] - ax.ax_profile.set_xlim(self.zmin, self.zmax) - ax.ax_profile.set_ylim(self.ymin, self.ymax) - ax.ax_profile.set_xlabel('dB') - ax.ax_profile.grid(b=True, axis='x') - ax.plt_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y, - color="k", linestyle="dashed", lw=2)[0] - [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()] else: - ax.plt.set_array(z[n].T.ravel()) - ax.plt_dop.set_data(mean[n], y) - if self.showprofile: - ax.plt_profile.set_data(self.data['rti'][self.max_time][n], y) - ax.plt_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y) + ax.plt.set_array(spc[pair[1]].T.ravel()) + self.titles.append('CH {}: {:3.2f}dB'.format(n, noise)) + + out = cspc[n]/numpy.sqrt(spc[pair[0]]*spc[pair[1]]) + coh = numpy.abs(out) + phase = numpy.arctan2(out.imag, out.real)*180/numpy.pi + + ax = self.axes[4*n+2] + if ax.firsttime: + ax.plt = ax.pcolormesh(x, y, coh.T, + vmin=0, + vmax=1, + cmap=plt.get_cmap(self.colormap_coh) + ) + else: + ax.plt.set_array(coh.T.ravel()) + self.titles.append('Coherence Ch{} * Ch{}'.format(pair[0], pair[1])) - ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]), - size=8) + ax = self.axes[4*n+3] + if ax.firsttime: + ax.plt = ax.pcolormesh(x, y, phase.T, + vmin=-180, + vmax=180, + cmap=plt.get_cmap(self.colormap_phase) + ) + else: + ax.plt.set_array(phase.T.ravel()) + self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1])) + self.saveTime = self.max_time +class PlotSpectraMeanData(PlotSpectraData): + ''' + Plot for Spectra and Mean + ''' + CODE = 'spc_mean' + colormap = 'jro' + + class PlotRTIData(PlotData): + ''' + Plot for RTI data + ''' CODE = 'rti' colormap = 'jro' def setup(self): - self.ncols = 1 - self.nrows = self.dataOut.nChannels - self.width = 10 - #TODO : arreglar la altura de la figura, esta hardcodeada. - #Se arreglo, testear! - if self.ind_plt_ch: - self.height = 3.2#*self.nrows if self.nrows<6 else 12 - else: - self.height = 2.2*self.nrows if self.nrows<6 else 12 - - ''' - if self.nrows==1: - self.height += 1 - ''' + self.xaxis = 'time' + self.ncols = 1 + self.nrows = len(self.data.channels) + self.nplots = len(self.data.channels) self.ylabel = 'Range [Km]' - self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList] - - ''' - Logica: - 1) Si la variable ind_plt_ch es True, va a crear mas de 1 figura - 2) guardamos "Figures" en una lista y "axes" en otra, quizas se deberia guardar el - axis dentro de "Figures" como un diccionario. - ''' - if self.ind_plt_ch is False: #standard mode - - if self.figure is None: #solo para la priemra vez - self.figure = plt.figure(figsize=(self.width, self.height), - edgecolor='k', - facecolor='w') - else: - self.figure.clf() - self.axes = [] - - - for n in range(self.nrows): - ax = self.figure.add_subplot(self.nrows, self.ncols, n+1) - #ax = self.figure(n+1) - ax.firsttime = True - self.axes.append(ax) - - else : #append one figure foreach channel in channelList - if self.figurelist == None: - self.figurelist = [] - for n in range(self.nrows): - self.figure = plt.figure(figsize=(self.width, self.height), - edgecolor='k', - facecolor='w') - #add always one subplot - self.figurelist.append(self.figure) - - else : # cada dia nuevo limpia el axes, pero mantiene el figure - for eachfigure in self.figurelist: - eachfigure.clf() # eliminaria todas las figuras de la lista? - self.axes = [] - - for eachfigure in self.figurelist: - ax = eachfigure.add_subplot(1,1,1) #solo 1 axis por figura - #ax = self.figure(n+1) - ax.firsttime = True - #Cada figura tiene un distinto puntero - self.axes.append(ax) - #plt.close(eachfigure) - + self.cb_label = 'dB' + self.titles = ['{} Channel {}'.format(self.CODE.upper(), x) for x in range(self.nrows)] def plot(self): + self.x = self.data.times + self.y = self.data.heights + self.z = self.data[self.CODE] + self.z = numpy.ma.masked_invalid(self.z) - if self.ind_plt_ch is False: #standard mode - self.x = np.array(self.times) - self.y = self.dataOut.getHeiRange() - self.z = [] - - for ch in range(self.nrows): - self.z.append([self.data[self.CODE][t][ch] for t in self.times]) - - self.z = np.array(self.z) - for n, ax in enumerate(self.axes): - x, y, z = self.fill_gaps(*self.decimate()) - xmin = self.min_time - xmax = xmin+self.xrange*60*60 - self.zmin = self.zmin if self.zmin else np.min(self.z) - self.zmax = self.zmax if self.zmax else np.max(self.z) - if ax.firsttime: - self.ymin = self.ymin if self.ymin else np.nanmin(self.y) - self.ymax = self.ymax if self.ymax else np.nanmax(self.y) - plot = ax.pcolormesh(x, y, z[n].T, - vmin=self.zmin, - vmax=self.zmax, - cmap=plt.get_cmap(self.colormap) - ) - divider = make_axes_locatable(ax) - cax = divider.new_horizontal(size='2%', pad=0.05) - self.figure.add_axes(cax) - plt.colorbar(plot, cax) - ax.set_ylim(self.ymin, self.ymax) - ax.xaxis.set_major_formatter(FuncFormatter(func)) - ax.xaxis.set_major_locator(LinearLocator(6)) - ax.set_ylabel(self.ylabel) - if self.xmin is None: - xmin = self.min_time - else: - xmin = (datetime.datetime.combine(self.dataOut.datatime.date(), - datetime.time(self.xmin, 0, 0))-d1970).total_seconds() - ax.set_xlim(xmin, xmax) - ax.firsttime = False - else: - ax.collections.remove(ax.collections[0]) - ax.set_xlim(xmin, xmax) - plot = ax.pcolormesh(x, y, z[n].T, - vmin=self.zmin, - vmax=self.zmax, - cmap=plt.get_cmap(self.colormap) - ) - ax.set_title('{} {}'.format(self.titles[n], - datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')), - size=8) - - self.saveTime = self.min_time - else : - self.x = np.array(self.times) - self.y = self.dataOut.getHeiRange() - self.z = [] - - for ch in range(self.nrows): - self.z.append([self.data[self.CODE][t][ch] for t in self.times]) - - self.z = np.array(self.z) - for n, eachfigure in enumerate(self.figurelist): #estaba ax in axes - - x, y, z = self.fill_gaps(*self.decimate()) - xmin = self.min_time - xmax = xmin+self.xrange*60*60 - self.zmin = self.zmin if self.zmin else np.min(self.z) - self.zmax = self.zmax if self.zmax else np.max(self.z) - if self.axes[n].firsttime: - self.ymin = self.ymin if self.ymin else np.nanmin(self.y) - self.ymax = self.ymax if self.ymax else np.nanmax(self.y) - plot = self.axes[n].pcolormesh(x, y, z[n].T, - vmin=self.zmin, - vmax=self.zmax, - cmap=plt.get_cmap(self.colormap) - ) - divider = make_axes_locatable(self.axes[n]) - cax = divider.new_horizontal(size='2%', pad=0.05) - eachfigure.add_axes(cax) - #self.figure2.add_axes(cax) - plt.colorbar(plot, cax) - self.axes[n].set_ylim(self.ymin, self.ymax) - - self.axes[n].xaxis.set_major_formatter(FuncFormatter(func)) - self.axes[n].xaxis.set_major_locator(LinearLocator(6)) - - self.axes[n].set_ylabel(self.ylabel) - - if self.xmin is None: - xmin = self.min_time - else: - xmin = (datetime.datetime.combine(self.dataOut.datatime.date(), - datetime.time(self.xmin, 0, 0))-d1970).total_seconds() - - self.axes[n].set_xlim(xmin, xmax) - self.axes[n].firsttime = False - else: - self.axes[n].collections.remove(self.axes[n].collections[0]) - self.axes[n].set_xlim(xmin, xmax) - plot = self.axes[n].pcolormesh(x, y, z[n].T, - vmin=self.zmin, - vmax=self.zmax, - cmap=plt.get_cmap(self.colormap) - ) - self.axes[n].set_title('{} {}'.format(self.titles[n], - datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')), - size=8) + for n, ax in enumerate(self.axes): + x, y, z = self.fill_gaps(*self.decimate()) + self.zmin = self.zmin if self.zmin else numpy.min(self.z) + self.zmax = self.zmax if self.zmax else numpy.max(self.z) + if ax.firsttime: + ax.plt = ax.pcolormesh(x, y, z[n].T, + vmin=self.zmin, + vmax=self.zmax, + cmap=plt.get_cmap(self.colormap) + ) + if self.showprofile: + ax.plot_profile= self.pf_axes[n].plot(self.data['rti'][n][-1], self.y)[0] + ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y, + color="k", linestyle="dashed", lw=1)[0] + else: + ax.collections.remove(ax.collections[0]) + ax.plt = ax.pcolormesh(x, y, z[n].T, + vmin=self.zmin, + vmax=self.zmax, + cmap=plt.get_cmap(self.colormap) + ) + if self.showprofile: + ax.plot_profile.set_data(self.data['rti'][n][-1], self.y) + ax.plot_noise.set_data(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y) - self.saveTime = self.min_time + self.saveTime = self.min_time class PlotCOHData(PlotRTIData): + ''' + Plot for Coherence data + ''' CODE = 'coh' def setup(self): - + self.xaxis = 'time' self.ncols = 1 - self.nrows = self.dataOut.nPairs - self.width = 10 - self.height = 2.2*self.nrows if self.nrows<6 else 12 - self.ind_plt_ch = False #just for coherence and phase - if self.nrows==1: - self.height += 1 - self.ylabel = 'Range [Km]' - self.titles = ['{} Ch{} * Ch{}'.format(self.CODE.upper(), x[0], x[1]) for x in self.dataOut.pairsList] - - if self.figure is None: - self.figure = plt.figure(figsize=(self.width, self.height), - edgecolor='k', - facecolor='w') + self.nrows = len(self.data.pairs) + self.nplots = len(self.data.pairs) + self.ylabel = 'Range [Km]' + if self.CODE == 'coh': + self.cb_label = '' + self.titles = ['Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs] else: - self.figure.clf() - self.axes = [] + self.cb_label = 'Degrees' + self.titles = ['Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs] - for n in range(self.nrows): - ax = self.figure.add_subplot(self.nrows, self.ncols, n+1) - ax.firsttime = True - self.axes.append(ax) + +class PlotPHASEData(PlotCOHData): + ''' + Plot for Phase map data + ''' + + CODE = 'phase' + colormap = 'seismic' class PlotNoiseData(PlotData): + ''' + Plot for noise + ''' + CODE = 'noise' def setup(self): - + self.xaxis = 'time' self.ncols = 1 self.nrows = 1 - self.width = 10 - self.height = 3.2 + self.nplots = 1 self.ylabel = 'Intensity [dB]' self.titles = ['Noise'] - - if self.figure is None: - self.figure = plt.figure(figsize=(self.width, self.height), - edgecolor='k', - facecolor='w') - else: - self.figure.clf() - self.axes = [] - - self.ax = self.figure.add_subplot(self.nrows, self.ncols, 1) - self.ax.firsttime = True + self.colorbar = False def plot(self): - x = self.times + x = self.data.times xmin = self.min_time xmax = xmin+self.xrange*60*60 - if self.ax.firsttime: - for ch in self.dataOut.channelList: - y = [self.data[self.CODE][t][ch] for t in self.times] - self.ax.plot(x, y, lw=1, label='Ch{}'.format(ch)) - self.ax.firsttime = False - self.ax.xaxis.set_major_formatter(FuncFormatter(func)) - self.ax.xaxis.set_major_locator(LinearLocator(6)) - self.ax.set_ylabel(self.ylabel) + Y = self.data[self.CODE] + + if self.axes[0].firsttime: + for ch in self.data.channels: + y = Y[ch] + self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch)) plt.legend() else: - for ch in self.dataOut.channelList: - y = [self.data[self.CODE][t][ch] for t in self.times] - self.ax.lines[ch].set_data(x, y) - - self.ax.set_xlim(xmin, xmax) - self.ax.set_ylim(min(y)-5, max(y)+5) + for ch in self.data.channels: + y = Y[ch] + self.axes[0].lines[ch].set_data(x, y) + + self.ymin = numpy.nanmin(Y) - 5 + self.ymax = numpy.nanmax(Y) + 5 self.saveTime = self.min_time -class PlotWindProfilerData(PlotRTIData): - - CODE = 'wind' - colormap = 'seismic' - - def setup(self): - self.ncols = 1 - self.nrows = self.dataOut.data_output.shape[0] - self.width = 10 - self.height = 2.2*self.nrows - self.ylabel = 'Height [Km]' - self.titles = ['Zonal Wind' ,'Meridional Wind', 'Vertical Wind'] - self.clabels = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)'] - self.windFactor = [1, 1, 100] - - if self.figure is None: - self.figure = plt.figure(figsize=(self.width, self.height), - edgecolor='k', - facecolor='w') - else: - self.figure.clf() - self.axes = [] - - for n in range(self.nrows): - ax = self.figure.add_subplot(self.nrows, self.ncols, n+1) - ax.firsttime = True - self.axes.append(ax) - - def plot(self): - - self.x = np.array(self.times) - self.y = self.dataOut.heightList - self.z = [] - - for ch in range(self.nrows): - self.z.append([self.data['output'][t][ch] for t in self.times]) - - self.z = np.array(self.z) - self.z = numpy.ma.masked_invalid(self.z) - - cmap=plt.get_cmap(self.colormap) - cmap.set_bad('black', 1.) - - for n, ax in enumerate(self.axes): - x, y, z = self.fill_gaps(*self.decimate()) - xmin = self.min_time - xmax = xmin+self.xrange*60*60 - if ax.firsttime: - self.ymin = self.ymin if self.ymin else np.nanmin(self.y) - self.ymax = self.ymax if self.ymax else np.nanmax(self.y) - self.zmax = self.zmax if self.zmax else numpy.nanmax(abs(self.z[:-1, :])) - self.zmin = self.zmin if self.zmin else -self.zmax - - plot = ax.pcolormesh(x, y, z[n].T*self.windFactor[n], - vmin=self.zmin, - vmax=self.zmax, - cmap=cmap - ) - divider = make_axes_locatable(ax) - cax = divider.new_horizontal(size='2%', pad=0.05) - self.figure.add_axes(cax) - cb = plt.colorbar(plot, cax) - cb.set_label(self.clabels[n]) - ax.set_ylim(self.ymin, self.ymax) - - ax.xaxis.set_major_formatter(FuncFormatter(func)) - ax.xaxis.set_major_locator(LinearLocator(6)) - - ax.set_ylabel(self.ylabel) - - ax.set_xlim(xmin, xmax) - ax.firsttime = False - else: - ax.collections.remove(ax.collections[0]) - ax.set_xlim(xmin, xmax) - plot = ax.pcolormesh(x, y, z[n].T*self.windFactor[n], - vmin=self.zmin, - vmax=self.zmax, - cmap=plt.get_cmap(self.colormap) - ) - ax.set_title('{} {}'.format(self.titles[n], - datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')), - size=8) - - self.saveTime = self.min_time - - class PlotSNRData(PlotRTIData): + ''' + Plot for SNR Data + ''' + CODE = 'snr' colormap = 'jet' + class PlotDOPData(PlotRTIData): + ''' + Plot for DOPPLER Data + ''' + CODE = 'dop' colormap = 'jet' -class PlotPHASEData(PlotCOHData): - CODE = 'phase' - colormap = 'seismic' - - class PlotSkyMapData(PlotData): + ''' + Plot for meteors detection data + ''' CODE = 'met' @@ -923,7 +681,7 @@ class PlotSkyMapData(PlotData): def plot(self): - arrayParameters = np.concatenate([self.data['param'][t] for t in self.times]) + arrayParameters = numpy.concatenate([self.data['param'][t] for t in self.data.times]) error = arrayParameters[:,-1] indValid = numpy.where(error == 0)[0] finalMeteor = arrayParameters[indValid,:] @@ -953,3 +711,72 @@ class PlotSkyMapData(PlotData): self.ax.set_title(title, size=8) self.saveTime = self.max_time + +class PlotParamData(PlotRTIData): + ''' + Plot for data_param object + ''' + + CODE = 'param' + colormap = 'seismic' + + def setup(self): + self.xaxis = 'time' + self.ncols = 1 + self.nrows = self.data.shape(self.CODE)[0] + self.nplots = self.nrows + if self.showSNR: + self.nrows += 1 + + self.ylabel = 'Height [Km]' + self.titles = self.data.parameters \ + if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)] + if self.showSNR: + self.titles.append('SNR') + + def plot(self): + self.data.normalize_heights() + self.x = self.data.times + self.y = self.data.heights + if self.showSNR: + self.z = numpy.concatenate( + (self.data[self.CODE], self.data['snr']) + ) + else: + self.z = self.data[self.CODE] + + self.z = numpy.ma.masked_invalid(self.z) + + for n, ax in enumerate(self.axes): + + x, y, z = self.fill_gaps(*self.decimate()) + + if ax.firsttime: + if self.zlimits is not None: + self.zmin, self.zmax = self.zlimits[n] + self.zmax = self.zmax if self.zmax is not None else numpy.nanmax(abs(self.z[:-1, :])) + self.zmin = self.zmin if self.zmin is not None else -self.zmax + ax.plt = ax.pcolormesh(x, y, z[n, :, :].T*self.factors[n], + vmin=self.zmin, + vmax=self.zmax, + cmap=self.cmaps[n] + ) + else: + if self.zlimits is not None: + self.zmin, self.zmax = self.zlimits[n] + ax.collections.remove(ax.collections[0]) + ax.plt = ax.pcolormesh(x, y, z[n, :, :].T*self.factors[n], + vmin=self.zmin, + vmax=self.zmax, + cmap=self.cmaps[n] + ) + + self.saveTime = self.min_time + +class PlotOuputData(PlotParamData): + ''' + Plot data_output object + ''' + + CODE = 'output' + colormap = 'seismic' \ No newline at end of file diff --git a/schainpy/model/graphics/jroplot_parameters.py b/schainpy/model/graphics/jroplot_parameters.py index b18b6a7..9167f8b 100644 --- a/schainpy/model/graphics/jroplot_parameters.py +++ b/schainpy/model/graphics/jroplot_parameters.py @@ -225,7 +225,6 @@ class MomentsPlot(Figure): WIDTHPROF = None HEIGHTPROF = None PREFIX = 'prm' - def __init__(self, **kwargs): Figure.__init__(self, **kwargs) self.isConfig = False @@ -1626,6 +1625,7 @@ class PhasePlot(Figure): PREFIX = 'mphase' + def __init__(self, **kwargs): Figure.__init__(self, **kwargs) self.timerange = 24*60*60 diff --git a/schainpy/model/graphics/jroplot_spectra.py b/schainpy/model/graphics/jroplot_spectra.py index e5b0d0d..eafac9f 100644 --- a/schainpy/model/graphics/jroplot_spectra.py +++ b/schainpy/model/graphics/jroplot_spectra.py @@ -10,6 +10,7 @@ import numpy from figure import Figure, isRealtime, isTimeInHourRange from plotting_codes import * + class SpectraPlot(Figure): isConfig = None diff --git a/schainpy/model/graphics/jroplotter.py b/schainpy/model/graphics/jroplotter.py index 1a632b0..118ca9b 100644 --- a/schainpy/model/graphics/jroplotter.py +++ b/schainpy/model/graphics/jroplotter.py @@ -124,7 +124,7 @@ class PlotManager(): subtitle = "%s:\n" %(name) subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname()) subtitle += "Working directory: %s\n" %os.path.abspath("./") -# subtitle += "Configuration file: %s\n" %self.filename + # subtitle += "Configuration file: %s\n" %self.filename subtitle += "Time: %s\n" %str(datetime.datetime.now()) adminObj = schainpy.admin.SchainNotify() diff --git a/schainpy/model/io/jroIO_amisr.py b/schainpy/model/io/jroIO_amisr.py index adf4e9d..3be5aaa 100644 --- a/schainpy/model/io/jroIO_amisr.py +++ b/schainpy/model/io/jroIO_amisr.py @@ -267,7 +267,7 @@ class AMISRReader(ProcessingUnit): self.dirnameList = new_dirnameList return 1 - def __searchFilesOnline(self, + def searchFilesOnLine(self, path, walk=True): @@ -287,7 +287,7 @@ class AMISRReader(ProcessingUnit): return - def __searchFilesOffline(self, + def searchFilesOffLine(self, path, startDate, endDate, @@ -494,9 +494,9 @@ class AMISRReader(ProcessingUnit): self.online = online if not(online): #Busqueda de archivos offline - self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk) + self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk) else: - self.__searchFilesOnline(path, walk) + self.searchFilesOnLine(path, walk) if not(self.filenameList): print "There is no files into the folder: %s"%(path) diff --git a/schainpy/model/io/jroIO_base.py b/schainpy/model/io/jroIO_base.py index 919bba0..816b353 100644 --- a/schainpy/model/io/jroIO_base.py +++ b/schainpy/model/io/jroIO_base.py @@ -542,7 +542,6 @@ class JRODataIO: class JRODataReader(JRODataIO): - online = 0 realtime = 0 @@ -579,7 +578,6 @@ class JRODataReader(JRODataIO): selBlocktime = None - def __init__(self): """ @@ -603,19 +601,18 @@ class JRODataReader(JRODataIO): raise NotImplementedError - def __searchFilesOffLine(self, - path, - startDate=None, - endDate=None, - startTime=datetime.time(0,0,0), - endTime=datetime.time(23,59,59), - set=None, - expLabel='', - ext='.r', - queue=None, - cursor=None, - skip=None, - walk=True): + def searchFilesOffLine(self, + path, + startDate=None, + endDate=None, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + set=None, + expLabel='', + ext='.r', + cursor=None, + skip=None, + walk=True): self.filenameList = [] self.datetimeList = [] @@ -625,8 +622,7 @@ class JRODataReader(JRODataIO): dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True) if dateList == []: -# print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path) - return None, None + return [], [] if len(dateList) > 1: print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList)) @@ -637,7 +633,6 @@ class JRODataReader(JRODataIO): datetimeList = [] for thisPath in pathList: -# thisPath = pathList[pathDict[file]] fileList = glob.glob1(thisPath, "*%s" %ext) fileList.sort() @@ -645,10 +640,8 @@ class JRODataReader(JRODataIO): skippedFileList = [] if cursor is not None and skip is not None: - # if cursor*skip > len(fileList): + if skip == 0: - if queue is not None: - queue.put(len(fileList)) skippedFileList = [] else: skippedFileList = fileList[cursor*skip: cursor*skip + skip] @@ -673,13 +666,13 @@ class JRODataReader(JRODataIO): if not(filenameList): print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path) - return None, None + return [], [] print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime) print - for i in range(len(filenameList)): - print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) + # for i in range(len(filenameList)): + # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) self.filenameList = filenameList self.datetimeList = datetimeList @@ -1043,8 +1036,7 @@ class JRODataReader(JRODataIO): #Skip block out of startTime and endTime while True: - if not(self.__setNewBlock()): - print 'returning' + if not(self.__setNewBlock()): return 0 if not(self.readBlock()): @@ -1273,7 +1265,6 @@ class JRODataReader(JRODataIO): realtime=False, blocksize=None, blocktime=None, - queue=None, skip=None, cursor=None, warnings=True, @@ -1322,19 +1313,13 @@ class JRODataReader(JRODataIO): last_set = None else: print "[Reading] Searching files in offline mode ..." - pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate, + pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate, startTime=startTime, endTime=endTime, set=set, expLabel=expLabel, ext=ext, walk=walk, cursor=cursor, - skip=skip, queue=queue) + skip=skip) if not(pathList): - # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path, - # datetime.datetime.combine(startDate,startTime).ctime(), - # datetime.datetime.combine(endDate,endTime).ctime()) - - # sys.exit(-1) - self.fileIndex = -1 self.pathList = [] self.filenameList = [] @@ -1451,7 +1436,6 @@ class JRODataReader(JRODataIO): self.__printInfo = False - def run(self, path=None, startDate=None, @@ -1469,7 +1453,6 @@ class JRODataReader(JRODataIO): realtime=False, blocksize=None, blocktime=None, - queue=None, skip=None, cursor=None, warnings=True, @@ -1477,29 +1460,27 @@ class JRODataReader(JRODataIO): verbose=True, **kwargs): if not(self.isConfig): -# self.dataOut = dataOut - self.setup( path=path, - startDate=startDate, - endDate=endDate, - startTime=startTime, - endTime=endTime, - set=set, - expLabel=expLabel, - ext=ext, - online=online, - delay=delay, - walk=walk, - getblock=getblock, - nTxs=nTxs, - realtime=realtime, - blocksize=blocksize, - blocktime=blocktime, - queue=queue, - skip=skip, - cursor=cursor, - warnings=warnings, - server=server, - verbose=verbose) + self.setup(path=path, + startDate=startDate, + endDate=endDate, + startTime=startTime, + endTime=endTime, + set=set, + expLabel=expLabel, + ext=ext, + online=online, + delay=delay, + walk=walk, + getblock=getblock, + nTxs=nTxs, + realtime=realtime, + blocksize=blocksize, + blocktime=blocktime, + skip=skip, + cursor=cursor, + warnings=warnings, + server=server, + verbose=verbose) self.isConfig = True if server is None: self.getData() diff --git a/schainpy/model/io/jroIO_heispectra.py b/schainpy/model/io/jroIO_heispectra.py index 06b1037..ddbffab 100644 --- a/schainpy/model/io/jroIO_heispectra.py +++ b/schainpy/model/io/jroIO_heispectra.py @@ -119,7 +119,6 @@ class Metadata(object): self.parmConfObjList.append(parmConfObj) class FitsWriter(Operation): - def __init__(self, **kwargs): Operation.__init__(self, **kwargs) self.isConfig = False @@ -276,9 +275,9 @@ class FitsWriter(Operation): self.setNextFile() self.writeNextBlock() - def run(self, dataOut, **kwargs): + def run(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None, **kwargs): if not(self.isConfig): - self.setup(dataOut, **kwargs) + self.setup(dataOut, path, dataBlocksPerFile=dataBlocksPerFile, metadatafile=metadatafile, **kwargs) self.isConfig = True self.putData() @@ -454,7 +453,7 @@ class FitsReader(ProcessingUnit): # self.blockIndex = 1 return 1 - def __searchFilesOffLine(self, + def searchFilesOffLine(self, path, startDate, endDate, @@ -560,7 +559,7 @@ class FitsReader(ProcessingUnit): if not(online): print "Searching files in offline mode ..." - pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate, + pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate, startTime=startTime, endTime=endTime, set=set, expLabel=expLabel, ext=ext, walk=walk) diff --git a/schainpy/model/io/jroIO_hf.py b/schainpy/model/io/jroIO_hf.py index 690492a..5ba1787 100644 --- a/schainpy/model/io/jroIO_hf.py +++ b/schainpy/model/io/jroIO_hf.py @@ -415,7 +415,7 @@ class HFReader(ProcessingUnit): - def __searchFilesOffline(self, + def searchFilesOffLine(self, path, startDate, endDate, @@ -438,7 +438,7 @@ class HFReader(ProcessingUnit): return - def __searchFilesOnline(self, + def searchFilesOnLine(self, path, expLabel= "", ext=None, @@ -636,10 +636,10 @@ class HFReader(ProcessingUnit): if not(online): print "Searching files in offline mode..." - self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk) + self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk) else: print "Searching files in online mode..." - self.__searchFilesOnline(path, walk,ext,set=set) + self.searchFilesOnLine(path, walk,ext,set=set) if set==None: pass else: @@ -647,7 +647,7 @@ class HFReader(ProcessingUnit): # for nTries in range(self.nTries): # -# fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set) +# fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set) # # if fullpath: # break diff --git a/schainpy/model/io/jroIO_kamisr.py b/schainpy/model/io/jroIO_kamisr.py index 9fcbf1c..7e4d14a 100644 --- a/schainpy/model/io/jroIO_kamisr.py +++ b/schainpy/model/io/jroIO_kamisr.py @@ -106,9 +106,9 @@ class AMISRReader(ProcessingUnit): #self.findFiles() if not(online): #Busqueda de archivos offline - self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk) + self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk) else: - self.__searchFilesOnline(path, startDate, endDate, startTime,endTime,walk) + self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk) if not(self.filenameList): print "There is no files into the folder: %s"%(path) @@ -329,7 +329,7 @@ class AMISRReader(ProcessingUnit): self.dirnameList = new_dirnameList return 1 - def __searchFilesOnline(self, path, startDate, endDate, startTime=datetime.time(0,0,0), + def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59),walk=True): if endDate ==None: @@ -349,7 +349,7 @@ class AMISRReader(ProcessingUnit): return - def __searchFilesOffline(self, + def searchFilesOffLine(self, path, startDate, endDate, diff --git a/schainpy/model/io/jroIO_param.py b/schainpy/model/io/jroIO_param.py index 32922bf..fa4495c 100644 --- a/schainpy/model/io/jroIO_param.py +++ b/schainpy/model/io/jroIO_param.py @@ -97,7 +97,7 @@ class ParamReader(ProcessingUnit): self.timezone = 'lt' print "[Reading] Searching files in offline mode ..." - pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate, + pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate, startTime=startTime, endTime=endTime, ext=ext, walk=walk) @@ -115,7 +115,7 @@ class ParamReader(ProcessingUnit): return - def __searchFilesOffLine(self, + def searchFilesOffLine(self, path, startDate=None, endDate=None, @@ -607,29 +607,18 @@ class ParamWriter(Operation): self.isConfig = False return - def setup(self, dataOut, **kwargs): - - self.path = kwargs['path'] - self.setType = kwargs.get('setType', None) - - if kwargs.has_key('blocksPerFile'): - self.blocksPerFile = kwargs['blocksPerFile'] - else: - self.blocksPerFile = 10 - - self.metadataList = kwargs['metadataList'] - self.dataList = kwargs['dataList'] + def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs): + self.path = path + self.blocksPerFile = blocksPerFile + self.metadataList = metadataList + self.dataList = dataList self.dataOut = dataOut - - if kwargs.has_key('mode'): - mode = kwargs['mode'] - - if type(mode) == int: - mode = numpy.zeros(len(self.dataList)) + mode - else: - mode = numpy.ones(len(self.dataList)) - self.mode = mode + + if self.mode is not None: + self.mode = numpy.zeros(len(self.dataList)) + mode + else: + self.mode = numpy.ones(len(self.dataList)) arrayDim = numpy.zeros((len(self.dataList),5)) @@ -1089,10 +1078,11 @@ class ParamWriter(Operation): self.fp.close() return - def run(self, dataOut, **kwargs): + def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs): if not(self.isConfig): - flagdata = self.setup(dataOut, **kwargs) + flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile, + metadataList=metadataList, dataList=dataList, mode=mode, **kwargs) if not(flagdata): return diff --git a/schainpy/model/io/jroIO_voltage.py b/schainpy/model/io/jroIO_voltage.py index 66ae509..a34c870 100644 --- a/schainpy/model/io/jroIO_voltage.py +++ b/schainpy/model/io/jroIO_voltage.py @@ -180,13 +180,13 @@ class VoltageReader(JRODataReader, ProcessingUnit): def getBlockDimension(self): """ - Obtiene la cantidad de puntos a leer por cada bloque de datos + Obtiene la cantidad de puntos a leer por cada bloque de datos + + Affected: + self.blocksize - Affected: - self.blocksize - - Return: - None + Return: + None """ pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels self.blocksize = pts2read @@ -195,26 +195,26 @@ class VoltageReader(JRODataReader, ProcessingUnit): def readBlock(self): """ - readBlock lee el bloque de datos desde la posicion actual del puntero del archivo - (self.fp) y actualiza todos los parametros relacionados al bloque de datos - (metadata + data). La data leida es almacenada en el buffer y el contador del buffer - es seteado a 0 - - Inputs: - None - - Return: - None - - Affected: - self.profileIndex - self.datablock - self.flagIsNewFile - self.flagIsNewBlock - self.nTotalBlocks - - Exceptions: - Si un bloque leido no es un bloque valido + readBlock lee el bloque de datos desde la posicion actual del puntero del archivo + (self.fp) y actualiza todos los parametros relacionados al bloque de datos + (metadata + data). La data leida es almacenada en el buffer y el contador del buffer + es seteado a 0 + + Inputs: + None + + Return: + None + + Affected: + self.profileIndex + self.datablock + self.flagIsNewFile + self.flagIsNewBlock + self.nTotalBlocks + + Exceptions: + Si un bloque leido no es un bloque valido """ # if self.server is not None: @@ -237,7 +237,7 @@ class VoltageReader(JRODataReader, ProcessingUnit): if self.waitDataBlock(pointer_location=current_pointer_location): junk = numpy.fromfile( self.fp, self.dtype, self.blocksize ) junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) ) -# return 0 + # return 0 #Dimensions : nChannels, nProfiles, nSamples @@ -267,15 +267,15 @@ class VoltageReader(JRODataReader, ProcessingUnit): #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj. -# self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt -# -# if self.radarControllerHeaderObj.code is not None: -# -# self.dataOut.nCode = self.radarControllerHeaderObj.nCode -# -# self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud -# -# self.dataOut.code = self.radarControllerHeaderObj.code + # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt + # + # if self.radarControllerHeaderObj.code is not None: + # + # self.dataOut.nCode = self.radarControllerHeaderObj.nCode + # + # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud + # + # self.dataOut.code = self.radarControllerHeaderObj.code self.dataOut.dtype = self.dtype @@ -340,8 +340,8 @@ class VoltageReader(JRODataReader, ProcessingUnit): self.dtype = datatype_str #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1) -# self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels) -# self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels) + # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels) + # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels) self.getBlockDimension() @@ -355,7 +355,7 @@ class VoltageReader(JRODataReader, ProcessingUnit): self.blockPointer = 0 block = self.receiver.recv() - + self.basicHeaderObj.read(block[self.blockPointer:]) self.blockPointer += self.basicHeaderObj.length self.systemHeaderObj.read(block[self.blockPointer:]) @@ -378,7 +378,7 @@ class VoltageReader(JRODataReader, ProcessingUnit): if self.waitDataBlock(pointer_location=current_pointer_location): junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize ) junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) ) -# return 0 + # return 0 #Dimensions : nChannels, nProfiles, nSamples @@ -408,39 +408,39 @@ class VoltageReader(JRODataReader, ProcessingUnit): def getData(self): """ - getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut - del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos - en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando - "readNextBlock" - - Ademas incrementa el contador del buffer "self.profileIndex" en 1. - - Return: - - Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex - es igual al total de perfiles leidos desde el archivo. - - Si self.getByBlock == False: - - self.dataOut.data = buffer[:, thisProfile, :] - - shape = [nChannels, nHeis] - - Si self.getByBlock == True: - - self.dataOut.data = buffer[:, :, :] - - shape = [nChannels, nProfiles, nHeis] - - Variables afectadas: - self.dataOut - self.profileIndex - - Affected: - self.dataOut - self.profileIndex - self.flagDiscontinuousBlock - self.flagIsNewBlock + getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut + del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos + en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando + "readNextBlock" + + Ademas incrementa el contador del buffer "self.profileIndex" en 1. + + Return: + + Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex + es igual al total de perfiles leidos desde el archivo. + + Si self.getByBlock == False: + + self.dataOut.data = buffer[:, thisProfile, :] + + shape = [nChannels, nHeis] + + Si self.getByBlock == True: + + self.dataOut.data = buffer[:, :, :] + + shape = [nChannels, nProfiles, nHeis] + + Variables afectadas: + self.dataOut + self.profileIndex + + Affected: + self.dataOut + self.profileIndex + self.flagDiscontinuousBlock + self.flagIsNewBlock """ if self.flagNoMoreFiles: self.dataOut.flagNoData = True @@ -462,30 +462,30 @@ class VoltageReader(JRODataReader, ProcessingUnit): if not self.getByBlock: """ - Return profile by profile + Return profile by profile - If nTxs > 1 then one profile is divided by nTxs and number of total - blocks is increased by nTxs (nProfiles *= nTxs) + If nTxs > 1 then one profile is divided by nTxs and number of total + blocks is increased by nTxs (nProfiles *= nTxs) """ self.dataOut.flagDataAsBlock = False self.dataOut.data = self.datablock[:,self.profileIndex,:] self.dataOut.profileIndex = self.profileIndex self.profileIndex += 1 - -# elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles: -# """ -# Return all block -# """ -# self.dataOut.flagDataAsBlock = True -# self.dataOut.data = self.datablock -# self.dataOut.profileIndex = self.dataOut.nProfiles - 1 -# -# self.profileIndex = self.dataOut.nProfiles - + + # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles: + # """ + # Return all block + # """ + # self.dataOut.flagDataAsBlock = True + # self.dataOut.data = self.datablock + # self.dataOut.profileIndex = self.dataOut.nProfiles - 1 + # + # self.profileIndex = self.dataOut.nProfiles + else: """ - Return a block + Return a block """ if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles if self.selBlocktime != None: diff --git a/schainpy/model/proc/extensions.c b/schainpy/model/proc/extensions.c index 27f0c9d..8f74614 100644 --- a/schainpy/model/proc/extensions.c +++ b/schainpy/model/proc/extensions.c @@ -1,12 +1,22 @@ +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#define NUM_CPY_THREADS 8 #include #include #include +#include +#include +// void printArr(int *array); static PyObject *hildebrand_sekhon(PyObject *self, PyObject *args); +static PyObject *correlateByBlock(PyObject *self, PyObject *args); +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif static PyMethodDef extensionsMethods[] = { - { "hildebrand_sekhon", (PyCFunction)hildebrand_sekhon, METH_VARARGS, "get noise with" }, - { NULL, NULL, 0, NULL } + { "correlateByBlock", (PyCFunction)correlateByBlock, METH_VARARGS, "get correlation by block" }, + { "hildebrand_sekhon", (PyCFunction)hildebrand_sekhon, METH_VARARGS, "get noise with hildebrand_sekhon" }, + { NULL, NULL, 0, NULL } }; PyMODINIT_FUNC initcSchain() { @@ -14,13 +24,83 @@ PyMODINIT_FUNC initcSchain() { import_array(); } +static PyObject *correlateByBlock(PyObject *self, PyObject *args) { + + // int *x = (int*) malloc(4000000 * 216 * sizeof(int));; + // int a = 5; + // x = &a; + // int b = 6; + // x = &b; + // printf("Antes de imprimir x \n"); + // printf("%d \n", x[0]); + + PyObject *data_obj1, *data_obj2; + PyArrayObject *data_array1, *data_array2, *correlateRow, *out, *dataRow, *codeRow; //, , + int mode; + + if (!PyArg_ParseTuple(args, "OOi", &data_obj1, &data_obj2, &mode)) return NULL; + + data_array1 = (PyArrayObject *) PyArray_FROM_OTF(data_obj1, NPY_COMPLEX128, NPY_ARRAY_DEFAULT); + data_array2 = (PyArrayObject *) PyArray_FROM_OTF(data_obj2, NPY_FLOAT64, NPY_ARRAY_DEFAULT); + + npy_intp dims[1]; + dims[0] = 200; + npy_intp dims_code[1]; + dims_code[0] = 16; + + double complex * dataRaw; + double * codeRaw; + dataRaw = (double complex*) PyArray_DATA(data_array1); + codeRaw = (double *) PyArray_DATA(data_array2); + double complex ** outC = malloc(40000*200*sizeof(double complex)); + int i; + + clock_t start = clock(); + for(i=0; i<40000; i++){ + // codeRow = PyArray_SimpleNewFromData(1, dims_code, NPY_FLOAT64, codeRaw + 16 * i); + // dataRow = PyArray_SimpleNewFromData(1, dims, NPY_COMPLEX128, dataRaw + 200 * i); + // Py_INCREF(codeRow); + // Py_INCREF(dataRow); + // PyArray_ENABLEFLAGS(codeRow, NPY_ARRAY_OWNDATA); + // PyArray_ENABLEFLAGS(dataRow, NPY_ARRAY_OWNDATA); + correlateRow = (PyArrayObject *) PyArray_Correlate2(PyArray_SimpleNewFromData(1, dims_code, NPY_FLOAT64, codeRaw + 16 * i), PyArray_SimpleNewFromData(1, dims, NPY_COMPLEX128, dataRaw + 200 * i), (npy_intp) 2); + //Py_INCREF(correlateRow); + // PyArray_ENABLEFLAGS(correlateRow, NPY_ARRAY_OWNDATA); + memcpy(outC + 200*i, (double complex*) PyArray_DATA(correlateRow), 200 * sizeof(double complex)); + + Py_DECREF(correlateRow); + // Py_DECREF(codeRow); + // Py_DECREF(dataRow); + } + clock_t end = clock(); + float seconds = (float)(end - start) / CLOCKS_PER_SEC; + printf("%f", seconds); + // + npy_intp dimsret[2]; + dimsret[0] = 40000; + dimsret[1] = 200; + out = PyArray_SimpleNewFromData(2, dimsret, NPY_COMPLEX128, outC); + PyArray_ENABLEFLAGS(out, NPY_ARRAY_OWNDATA); + //Py_INCREF(out); + Py_DECREF(data_array1); + Py_DECREF(data_array2); + // PyArray_DebugPrint(out); + // Py_DECREF(data_obj2); + // Py_DECREF(data_obj1); + // Py_DECREF(codeRow); + // Py_DECREF(dataRow); + // free(dataRaw); + // free(codeRaw); + + return PyArray_Return(out); +} + static PyObject *hildebrand_sekhon(PyObject *self, PyObject *args) { - /* Do your stuff here. */ double navg; PyObject *data_obj, *data_array; if (!PyArg_ParseTuple(args, "Od", &data_obj, &navg)) return NULL; - data_array = PyArray_FROM_OTF(data_obj, NPY_FLOAT64, NPY_IN_ARRAY); + data_array = PyArray_FROM_OTF(data_obj, NPY_FLOAT64, NPY_ARRAY_DEFAULT); if (data_array == NULL) { Py_XDECREF(data_array); Py_XDECREF(data_obj); @@ -56,3 +136,4 @@ static PyObject *hildebrand_sekhon(PyObject *self, PyObject *args) { return Py_BuildValue("d", lnoise); } + diff --git a/schainpy/model/proc/jroproc_base.py b/schainpy/model/proc/jroproc_base.py index 55f3afd..3b87703 100644 --- a/schainpy/model/proc/jroproc_base.py +++ b/schainpy/model/proc/jroproc_base.py @@ -200,7 +200,6 @@ class ProcessingUnit(object): return True def call(self, opType, opName=None, opId=None): - """ Return True si ejecuta la operacion interna nombrada "opName" o la operacion externa identificada con el id "opId"; con los argumentos "**kwargs". diff --git a/schainpy/model/proc/jroproc_heispectra.py b/schainpy/model/proc/jroproc_heispectra.py index e9a25fc..1c6976f 100644 --- a/schainpy/model/proc/jroproc_heispectra.py +++ b/schainpy/model/proc/jroproc_heispectra.py @@ -163,7 +163,6 @@ class IncohInt4SpectraHeis(Operation): n = None - def __init__(self, **kwargs): Operation.__init__(self, **kwargs) @@ -324,10 +323,10 @@ class IncohInt4SpectraHeis(Operation): return avgdata, avgdatatime - def run(self, dataOut, **kwargs): + def run(self, dataOut, n=None, timeInterval=None, overlapping=False, **kwargs): if not self.isConfig: - self.setup(**kwargs) + self.setup(n=n, timeInterval=timeInterval, overlapping=overlapping) self.isConfig = True avgdata, avgdatatime = self.integrate(dataOut.data_spc, dataOut.utctime) diff --git a/schainpy/model/proc/jroproc_parameters.py b/schainpy/model/proc/jroproc_parameters.py index e0ccf48..06822dc 100644 --- a/schainpy/model/proc/jroproc_parameters.py +++ b/schainpy/model/proc/jroproc_parameters.py @@ -52,13 +52,6 @@ def _unpickle_method(func_name, obj, cls): break return func.__get__(obj, cls) - - - - - - - class ParametersProc(ProcessingUnit): nSeconds = None @@ -104,9 +97,7 @@ class ParametersProc(ProcessingUnit): self.dataOut.timeInterval1 = self.dataIn.timeInterval self.dataOut.heightList = self.dataIn.getHeiRange() self.dataOut.frequency = self.dataIn.frequency - self.dataOut.noise = self.dataIn.noise - - + # self.dataOut.noise = self.dataIn.noise def run(self): @@ -125,22 +116,23 @@ class ParametersProc(ProcessingUnit): if self.dataIn.type == "Spectra": - self.dataOut.data_pre = (self.dataIn.data_spc,self.dataIn.data_cspc) - print 'self.dataIn.data_spc', self.dataIn.data_spc.shape + self.dataOut.data_pre = (self.dataIn.data_spc, self.dataIn.data_cspc) + self.dataOut.data_spc = self.dataIn.data_spc + self.dataOut.data_cspc = self.dataIn.data_cspc + self.dataOut.nProfiles = self.dataIn.nProfiles + self.dataOut.nIncohInt = self.dataIn.nIncohInt + self.dataOut.nFFTPoints = self.dataIn.nFFTPoints + self.dataOut.ippFactor = self.dataIn.ippFactor self.dataOut.abscissaList = self.dataIn.getVelRange(1) self.dataOut.spc_noise = self.dataIn.getNoise() - self.dataOut.spc_range = (self.dataIn.getFreqRange(1)/1000. , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1) ) - - self.dataOut.normFactor = self.dataIn.normFactor - #self.dataOut.outputInterval = self.dataIn.outputInterval + self.dataOut.spc_range = (self.dataIn.getFreqRange(1)/1000. , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1)) + self.dataOut.pairsList = self.dataIn.pairsList self.dataOut.groupList = self.dataIn.pairsList self.dataOut.flagNoData = False - #print 'datain chandist ',self.dataIn.ChanDist + if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels self.dataOut.ChanDist = self.dataIn.ChanDist - else: self.dataOut.ChanDist = None - - print 'datain chandist ',self.dataOut.ChanDist + else: self.dataOut.ChanDist = None if hasattr(self.dataIn, 'VelRange'): #Velocities range self.dataOut.VelRange = self.dataIn.VelRange diff --git a/schainpy/model/proc/jroproc_spectra.py b/schainpy/model/proc/jroproc_spectra.py index 4642b8a..5df7042 100644 --- a/schainpy/model/proc/jroproc_spectra.py +++ b/schainpy/model/proc/jroproc_spectra.py @@ -1,3 +1,5 @@ +import itertools + import numpy from jroproc_base import ProcessingUnit, Operation @@ -109,7 +111,10 @@ class SpectraProc(ProcessingUnit): if self.dataIn.type == "Spectra": self.dataOut.copy(self.dataIn) -# self.__selectPairs(pairsList) + if not pairsList: + pairsList = itertools.combinations(self.dataOut.channelList, 2) + if self.dataOut.data_cspc is not None: + self.__selectPairs(pairsList) return True if self.dataIn.type == "Voltage": @@ -178,27 +183,21 @@ class SpectraProc(ProcessingUnit): def __selectPairs(self, pairsList): - if channelList == None: + if not pairsList: return - pairsIndexListSelected = [] - - for thisPair in pairsList: + pairs = [] + pairsIndex = [] - if thisPair not in self.dataOut.pairsList: + for pair in pairsList: + if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList: continue - - pairIndex = self.dataOut.pairsList.index(thisPair) - - pairsIndexListSelected.append(pairIndex) - - if not pairsIndexListSelected: - self.dataOut.data_cspc = None - self.dataOut.pairsList = [] - return - - self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndexListSelected] - self.dataOut.pairsList = [self.dataOut.pairsList[i] for i in pairsIndexListSelected] + pairs.append(pair) + pairsIndex.append(pairs.index(pair)) + + self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex] + self.dataOut.pairsList = pairs + self.dataOut.pairsIndexList = pairsIndex return diff --git a/schainpy/model/proc/jroproc_voltage.py b/schainpy/model/proc/jroproc_voltage.py index 5cbf451..61c6ed9 100644 --- a/schainpy/model/proc/jroproc_voltage.py +++ b/schainpy/model/proc/jroproc_voltage.py @@ -1,9 +1,10 @@ import sys import numpy from scipy import interpolate - +from schainpy import cSchain from jroproc_base import ProcessingUnit, Operation from schainpy.model.data.jrodata import Voltage +from time import time class VoltageProc(ProcessingUnit): @@ -332,7 +333,6 @@ class CohInt(Operation): n = None - def __init__(self, **kwargs): Operation.__init__(self, **kwargs) @@ -345,10 +345,9 @@ class CohInt(Operation): Inputs: - n : Number of coherent integrations - timeInterval : Time of integration. If the parameter "n" is selected this one does not work - overlapping : - + n : Number of coherent integrations + timeInterval : Time of integration. If the parameter "n" is selected this one does not work + overlapping : """ self.__initime = None @@ -548,14 +547,13 @@ class Decoder(Operation): nCode = None nBaud = None - def __init__(self, **kwargs): Operation.__init__(self, **kwargs) self.times = None self.osamp = None -# self.__setValues = False + # self.__setValues = False self.isConfig = False def setup(self, code, osamp, dataOut): @@ -631,12 +629,12 @@ class Decoder(Operation): junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize)) junk = junk.flatten() code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud)) - - for i in range(self.__nChannels): - for j in range(self.__nProfiles): - self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:] - - return self.datadecTime + profilesList = xrange(self.__nProfiles) + + for i in range(self.__nChannels): + for j in profilesList: + self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:] + return self.datadecTime def __convolutionByBlockInFreq(self, data): @@ -653,6 +651,7 @@ class Decoder(Operation): return data + def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None): if dataOut.flagDecodeData: @@ -1087,7 +1086,6 @@ class SplitProfiles(Operation): dataOut.ippSeconds /= n class CombineProfiles(Operation): - def __init__(self, **kwargs): Operation.__init__(self, **kwargs) diff --git a/schainpy/model/utils/jroutils_ftp.py b/schainpy/model/utils/jroutils_ftp.py index 87ab15b..a507f2f 100644 --- a/schainpy/model/utils/jroutils_ftp.py +++ b/schainpy/model/utils/jroutils_ftp.py @@ -32,9 +32,7 @@ class Remote(Thread): None Written by: - - "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015 - + "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015 """ server = None @@ -168,13 +166,13 @@ class Remote(Thread): return 0 self.mutex.acquire() -# init = time.time() -# -# while(self.bussy): -# sleep(0.1) -# if time.time() - init > 2*self.period: -# return 0 - + # init = time.time() + # + # while(self.bussy): + # sleep(0.1) + # if time.time() - init > 2*self.period: + # return 0 + self.fileList = fileList self.mutex.release() return 1 @@ -197,8 +195,8 @@ class Remote(Thread): if self.stopFlag: break - -# self.bussy = True + + # self.bussy = True self.mutex.acquire() print "[Remote Server] Opening %s" %self.__server @@ -213,7 +211,7 @@ class Remote(Thread): self.close() self.mutex.release() -# self.bussy = False + # self.bussy = False print "[Remote Server] Thread stopped successfully" @@ -400,21 +398,21 @@ class SSHClient(Remote): def open(self, server, username, password, remotefolder, port=22): """ - This method is used to set SSH parameters and establish a connection to a remote server - - Inputs: - server - remote server IP Address - - username - remote server Username - - password - remote server password - - remotefolder - remote server current working directory - - Return: void - - Affects: - self.status - in case of error or fail connection this parameter is set to 0 else 1 + This method is used to set SSH parameters and establish a connection to a remote server + + Inputs: + server - remote server IP Address + + username - remote server Username + + password - remote server password + + remotefolder - remote server current working directory + + Return: void + + Affects: + self.status - in case of error or fail connection this parameter is set to 0 else 1 """ import socket @@ -440,7 +438,7 @@ class SSHClient(Remote): try: sshClientObj.connect(server, username=username, password=password, port=port) except paramiko.AuthenticationException, e: -# print "SSH username or password are incorrect: %s" + # print "SSH username or password are incorrect: %s" print "[SSH Server]:", e return 0 except SSHException, e: @@ -474,7 +472,7 @@ class SSHClient(Remote): def close(self): """ - Close connection to remote server + Close connection to remote server """ if not self.status: return 0 @@ -484,13 +482,13 @@ class SSHClient(Remote): def __execute(self, command): """ - __execute a command on remote server - - Input: - command - Exmaple 'ls -l' - - Return: - 0 in error case else 1 + __execute a command on remote server + + Input: + command - Exmaple 'ls -l' + + Return: + 0 in error case else 1 """ if not self.status: return 0 @@ -509,13 +507,13 @@ class SSHClient(Remote): def mkdir(self, remotefolder): """ - mkdir is used to make a new directory in remote server - - Input: - remotefolder - directory name - - Return: - 0 in error case else 1 + mkdir is used to make a new directory in remote server + + Input: + remotefolder - directory name + + Return: + 0 in error case else 1 """ command = 'mkdir %s' %remotefolder @@ -530,16 +528,16 @@ class SSHClient(Remote): def cd(self, remotefolder): """ - cd is used to change remote working directory on server - - Input: - remotefolder - current working directory - - Affects: - self.remotefolder - - Return: - 0 in case of error else 1 + cd is used to change remote working directory on server + + Input: + remotefolder - current working directory + + Affects: + self.remotefolder + + Return: + 0 in case of error else 1 """ if not self.status: return 0 @@ -582,8 +580,8 @@ class SendToServer(ProcessingUnit): ProcessingUnit.__init__(self, **kwargs) self.isConfig = False - self.clientObj = None - + self.clientObj = None + def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs): self.clientObj = None @@ -621,6 +619,7 @@ class SendToServer(ProcessingUnit): filenameList = glob.glob1(thisFolder, '*%s' %self.ext) if len(filenameList) < 1: + continue for thisFile in filenameList: @@ -707,9 +706,9 @@ class FTP(object): try: self.ftp = ftplib.FTP(self.server) self.ftp.login(self.username,self.password) - self.ftp.cwd(self.remotefolder) -# print 'Connect to FTP Server: Successfully' - + self.ftp.cwd(self.remotefolder) + # print 'Connect to FTP Server: Successfully' + except ftplib.all_errors: print 'Error FTP Service' self.status = 1 @@ -740,7 +739,7 @@ class FTP(object): name, ext = os.path.splitext(f) if ext != '': self.fileList.append(f) -# print 'filename: %s - size: %d'%(f,self.ftp.size(f)) + # print 'filename: %s - size: %d'%(f,self.ftp.size(f)) def parmsByDefault(self): server = 'jro-app.igp.gob.pe' diff --git a/schainpy/model/utils/jroutils_publish.py b/schainpy/model/utils/jroutils_publish.py index e46c8c9..726065a 100644 --- a/schainpy/model/utils/jroutils_publish.py +++ b/schainpy/model/utils/jroutils_publish.py @@ -7,7 +7,6 @@ import json import numpy import paho.mqtt.client as mqtt import zmq -from profilehooks import profile import datetime from zmq.utils.monitor import recv_monitor_message from functools import wraps @@ -16,6 +15,7 @@ from multiprocessing import Process from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit from schainpy.model.data.jrodata import JROData +from schainpy.utils import log MAXNUMX = 100 MAXNUMY = 100 @@ -31,14 +31,13 @@ def roundFloats(obj): return round(obj, 2) def decimate(z, MAXNUMY): - # dx = int(len(self.x)/self.__MAXNUMX) + 1 - dy = int(len(z[0])/MAXNUMY) + 1 return z[::, ::dy] class throttle(object): - """Decorator that prevents a function from being called more than once every + ''' + Decorator that prevents a function from being called more than once every time period. To create a function that cannot be called more than once a minute, but will sleep until it can be called: @@ -49,7 +48,7 @@ class throttle(object): for i in range(10): foo() print "This function has run %s times." % i - """ + ''' def __init__(self, seconds=0, minutes=0, hours=0): self.throttle_period = datetime.timedelta( @@ -73,9 +72,169 @@ class throttle(object): return wrapper +class Data(object): + ''' + Object to hold data to be plotted + ''' + + def __init__(self, plottypes, throttle_value): + self.plottypes = plottypes + self.throttle = throttle_value + self.ended = False + self.__times = [] + + def __str__(self): + dum = ['{}{}'.format(key, self.shape(key)) for key in self.data] + return 'Data[{}][{}]'.format(';'.join(dum), len(self.__times)) + + def __len__(self): + return len(self.__times) + + def __getitem__(self, key): + if key not in self.data: + raise KeyError(log.error('Missing key: {}'.format(key))) + + if 'spc' in key: + ret = self.data[key] + else: + ret = numpy.array([self.data[key][x] for x in self.times]) + if ret.ndim > 1: + ret = numpy.swapaxes(ret, 0, 1) + return ret + + def setup(self): + ''' + Configure object + ''' + + self.ended = False + self.data = {} + self.__times = [] + self.__heights = [] + self.__all_heights = set() + for plot in self.plottypes: + self.data[plot] = {} + + def shape(self, key): + ''' + Get the shape of the one-element data for the given key + ''' + + if len(self.data[key]): + if 'spc' in key: + return self.data[key].shape + return self.data[key][self.__times[0]].shape + return (0,) + + def update(self, dataOut): + ''' + Update data object with new dataOut + ''' + + tm = dataOut.utctime + if tm in self.__times: + return + + self.parameters = getattr(dataOut, 'parameters', []) + self.pairs = dataOut.pairsList + self.channels = dataOut.channelList + self.xrange = (dataOut.getFreqRange(1)/1000. , dataOut.getAcfRange(1) , dataOut.getVelRange(1)) + self.interval = dataOut.getTimeInterval() + self.__heights.append(dataOut.heightList) + self.__all_heights.update(dataOut.heightList) + self.__times.append(tm) + + for plot in self.plottypes: + if plot == 'spc': + z = dataOut.data_spc/dataOut.normFactor + self.data[plot] = 10*numpy.log10(z) + if plot == 'cspc': + self.data[plot] = dataOut.data_cspc + if plot == 'noise': + self.data[plot][tm] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor) + if plot == 'rti': + self.data[plot][tm] = dataOut.getPower() + if plot == 'snr_db': + self.data['snr'][tm] = dataOut.data_SNR + if plot == 'snr': + self.data[plot][tm] = 10*numpy.log10(dataOut.data_SNR) + if plot == 'dop': + self.data[plot][tm] = 10*numpy.log10(dataOut.data_DOP) + if plot == 'mean': + self.data[plot][tm] = dataOut.data_MEAN + if plot == 'std': + self.data[plot][tm] = dataOut.data_STD + if plot == 'coh': + self.data[plot][tm] = dataOut.getCoherence() + if plot == 'phase': + self.data[plot][tm] = dataOut.getCoherence(phase=True) + if plot == 'output': + self.data[plot][tm] = dataOut.data_output + if plot == 'param': + self.data[plot][tm] = dataOut.data_param + + def normalize_heights(self): + ''' + Ensure same-dimension of the data for different heighList + ''' + + H = numpy.array(list(self.__all_heights)) + H.sort() + for key in self.data: + shape = self.shape(key)[:-1] + H.shape + for tm, obj in self.data[key].items(): + h = self.__heights[self.__times.index(tm)] + if H.size == h.size: + continue + index = numpy.where(numpy.in1d(H, h))[0] + dummy = numpy.zeros(shape) + numpy.nan + if len(shape) == 2: + dummy[:, index] = obj + else: + dummy[index] = obj + self.data[key][tm] = dummy + + self.__heights = [H for tm in self.__times] + + def jsonify(self, decimate=False): + ''' + Convert data to json + ''' + + ret = {} + tm = self.times[-1] + + for key, value in self.data: + if key in ('spc', 'cspc'): + ret[key] = roundFloats(self.data[key].to_list()) + else: + ret[key] = roundFloats(self.data[key][tm].to_list()) + + ret['timestamp'] = tm + ret['interval'] = self.interval + + @property + def times(self): + ''' + Return the list of times of the current data + ''' + + ret = numpy.array(self.__times) + ret.sort() + return ret + + @property + def heights(self): + ''' + Return the list of heights of the current data + ''' + + return numpy.array(self.__heights[-1]) class PublishData(Operation): - """Clase publish.""" + ''' + Operation to send data over zmq. + ''' def __init__(self, **kwargs): """Inicio.""" @@ -87,11 +246,11 @@ class PublishData(Operation): def on_disconnect(self, client, userdata, rc): if rc != 0: - print("Unexpected disconnection.") + log.warning('Unexpected disconnection.') self.connect() def connect(self): - print 'trying to connect' + log.warning('trying to connect') try: self.client.connect( host=self.host, @@ -105,7 +264,7 @@ class PublishData(Operation): # retain=True # ) except: - print "MQTT Conection error." + log.error('MQTT Conection error.') self.client = False def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, verbose=True, **kwargs): @@ -120,8 +279,7 @@ class PublishData(Operation): self.zeromq = zeromq self.mqtt = kwargs.get('plottype', 0) self.client = None - self.verbose = verbose - self.dataOut.firstdata = True + self.verbose = verbose setup = [] if mqtt is 1: self.client = mqtt.Client( @@ -176,7 +334,6 @@ class PublishData(Operation): 'type': self.plottype, 'yData': yData } - # print payload elif self.plottype in ('rti', 'power'): data = getattr(self.dataOut, 'data_spc') @@ -230,15 +387,16 @@ class PublishData(Operation): 'timestamp': 'None', 'type': None } - # print 'Publishing data to {}'.format(self.host) + self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0) if self.zeromq is 1: if self.verbose: - print '[Sending] {} - {}'.format(self.dataOut.type, self.dataOut.datatime) + log.log( + '{} - {}'.format(self.dataOut.type, self.dataOut.datatime), + 'Sending' + ) self.zmq_socket.send_pyobj(self.dataOut) - self.dataOut.firstdata = False - def run(self, dataOut, **kwargs): self.dataOut = dataOut @@ -253,6 +411,7 @@ class PublishData(Operation): if self.zeromq is 1: self.dataOut.finished = True self.zmq_socket.send_pyobj(self.dataOut) + time.sleep(0.1) self.zmq_socket.close() if self.client: self.client.loop_stop() @@ -281,7 +440,7 @@ class ReceiverData(ProcessingUnit): self.receiver = self.context.socket(zmq.PULL) self.receiver.bind(self.address) time.sleep(0.5) - print '[Starting] ReceiverData from {}'.format(self.address) + log.success('ReceiverData from {}'.format(self.address)) def run(self): @@ -291,8 +450,9 @@ class ReceiverData(ProcessingUnit): self.isConfig = True self.dataOut = self.receiver.recv_pyobj() - print '[Receiving] {} - {}'.format(self.dataOut.type, - self.dataOut.datatime.ctime()) + log.log('{} - {}'.format(self.dataOut.type, + self.dataOut.datatime.ctime(),), + 'Receiving') class PlotterReceiver(ProcessingUnit, Process): @@ -306,7 +466,6 @@ class PlotterReceiver(ProcessingUnit, Process): self.mp = False self.isConfig = False self.isWebConfig = False - self.plottypes = [] self.connections = 0 server = kwargs.get('server', 'zmq.pipe') plot_server = kwargs.get('plot_server', 'zmq.web') @@ -326,19 +485,13 @@ class PlotterReceiver(ProcessingUnit, Process): self.realtime = kwargs.get('realtime', False) self.throttle_value = kwargs.get('throttle', 5) self.sendData = self.initThrottle(self.throttle_value) + self.dates = [] self.setup() def setup(self): - self.data = {} - self.data['times'] = [] - for plottype in self.plottypes: - self.data[plottype] = {} - self.data['noise'] = {} - self.data['throttle'] = self.throttle_value - self.data['ENDED'] = False - self.isConfig = True - self.data_web = {} + self.data = Data(self.plottypes, self.throttle_value) + self.isConfig = True def event_monitor(self, monitor): @@ -355,15 +508,13 @@ class PlotterReceiver(ProcessingUnit, Process): self.connections += 1 if evt['event'] == 512: pass - if self.connections == 0 and self.started is True: - self.ended = True evt.update({'description': events[evt['event']]}) if evt['event'] == zmq.EVENT_MONITOR_STOPPED: break monitor.close() - print("event monitor thread done!") + print('event monitor thread done!') def initThrottle(self, throttle_value): @@ -373,65 +524,16 @@ class PlotterReceiver(ProcessingUnit, Process): return sendDataThrottled - def send(self, data): - # print '[sending] data=%s size=%s' % (data.keys(), len(data['times'])) + log.success('Sending {}'.format(data), self.name) self.sender.send_pyobj(data) - - def update(self): - t = self.dataOut.utctime - - if t in self.data['times']: - return - - self.data['times'].append(t) - self.data['dataOut'] = self.dataOut - - for plottype in self.plottypes: - if plottype == 'spc': - z = self.dataOut.data_spc/self.dataOut.normFactor - self.data[plottype] = 10*numpy.log10(z) - self.data['noise'][t] = 10*numpy.log10(self.dataOut.getNoise()/self.dataOut.normFactor) - if plottype == 'cspc': - jcoherence = self.dataOut.data_cspc/numpy.sqrt(self.dataOut.data_spc*self.dataOut.data_spc) - self.data['cspc_coh'] = numpy.abs(jcoherence) - self.data['cspc_phase'] = numpy.arctan2(jcoherence.imag, jcoherence.real)*180/numpy.pi - if plottype == 'rti': - self.data[plottype][t] = self.dataOut.getPower() - if plottype == 'snr': - self.data[plottype][t] = 10*numpy.log10(self.dataOut.data_SNR) - if plottype == 'dop': - self.data[plottype][t] = 10*numpy.log10(self.dataOut.data_DOP) - if plottype == 'mean': - self.data[plottype][t] = self.dataOut.data_MEAN - if plottype == 'std': - self.data[plottype][t] = self.dataOut.data_STD - if plottype == 'coh': - self.data[plottype][t] = self.dataOut.getCoherence() - if plottype == 'phase': - self.data[plottype][t] = self.dataOut.getCoherence(phase=True) - if plottype == 'output': - self.data[plottype][t] = self.dataOut.data_output - if plottype == 'param': - self.data[plottype][t] = self.dataOut.data_param - if self.realtime: - self.data_web['timestamp'] = t - if plottype == 'spc': - self.data_web[plottype] = roundFloats(decimate(self.data[plottype]).tolist()) - elif plottype == 'cspc': - self.data_web['cspc_coh'] = roundFloats(decimate(self.data['cspc_coh']).tolist()) - self.data_web['cspc_phase'] = roundFloats(decimate(self.data['cspc_phase']).tolist()) - elif plottype == 'noise': - self.data_web['noise'] = roundFloats(self.data['noise'][t].tolist()) - else: - self.data_web[plottype] = roundFloats(decimate(self.data[plottype][t]).tolist()) - self.data_web['interval'] = self.dataOut.getTimeInterval() - self.data_web['type'] = plottype - def run(self): - print '[Starting] {} from {}'.format(self.name, self.address) + log.success( + 'Starting from {}'.format(self.address), + self.name + ) self.context = zmq.Context() self.receiver = self.context.socket(zmq.PULL) @@ -448,39 +550,39 @@ class PlotterReceiver(ProcessingUnit, Process): else: self.sender.bind("ipc:///tmp/zmq.plots") - time.sleep(3) + time.sleep(2) t = Thread(target=self.event_monitor, args=(monitor,)) t.start() while True: - self.dataOut = self.receiver.recv_pyobj() - # print '[Receiving] {} - {}'.format(self.dataOut.type, - # self.dataOut.datatime.ctime()) - - self.update() + dataOut = self.receiver.recv_pyobj() + dt = datetime.datetime.fromtimestamp(dataOut.utctime).date() + sended = False + if dt not in self.dates: + if self.data: + self.data.ended = True + self.send(self.data) + sended = True + self.data.setup() + self.dates.append(dt) - if self.dataOut.firstdata is True: - self.data['STARTED'] = True + self.data.update(dataOut) - if self.dataOut.finished is True: - self.send(self.data) + if dataOut.finished is True: self.connections -= 1 - if self.connections == 0 and self.started: - self.ended = True - self.data['ENDED'] = True + if self.connections == 0 and dt in self.dates: + self.data.ended = True self.send(self.data) - self.setup() - self.started = False + self.data.setup() else: if self.realtime: self.send(self.data) - self.sender_web.send_string(json.dumps(self.data_web)) + # self.sender_web.send_string(self.data.jsonify()) else: - self.sendData(self.send, self.data) - self.started = True + if not sended: + self.sendData(self.send, self.data) - self.data['STARTED'] = False return def sendToWeb(self): @@ -497,6 +599,6 @@ class PlotterReceiver(ProcessingUnit, Process): time.sleep(1) for kwargs in self.operationKwargs.values(): if 'plot' in kwargs: - print '[Sending] Config data to web for {}'.format(kwargs['code'].upper()) + log.success('[Sending] Config data to web for {}'.format(kwargs['code'].upper())) sender_web_config.send_string(json.dumps(kwargs)) - self.isWebConfig = True + self.isWebConfig = True \ No newline at end of file diff --git a/schainpy/scripts/JASMET30_MetDet.py b/schainpy/scripts/JASMET30_MetDet.py new file mode 100644 index 0000000..106a7a8 --- /dev/null +++ b/schainpy/scripts/JASMET30_MetDet.py @@ -0,0 +1,95 @@ + +import os, sys + +from schainpy.controller import Project + +controllerObj = Project() +controllerObj.setup(id = '002', name='script02', description="JASMET Meteor Detection") + +#-------------------------------------- Setup ----------------------------------------- +#Verificar estas variables + +#Path para los archivos +# path = '/mnt/jars/2016_08/NOCHE' +# path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/DIA' +# path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/NOCHE' +path = '/home/nanosat/data/jasmet' + +#Path para los graficos +pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/graphics') + +#Path para los archivos HDF5 de meteoros +pathfile = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/meteor') + +#Fechas para busqueda de archivos +startDate = '2010/08/29' +endDate = '2017/09/11' +#Horas para busqueda de archivos +startTime = '00:00:00' +endTime = '23:59:59' + + +#------------------------------ Voltage Reading Unit ---------------------------------- + +readUnitConfObj = controllerObj.addReadUnit(datatype='VoltageReader', + path=path, + startDate=startDate, + endDate=endDate, + startTime=startTime, + endTime=endTime, + online=0, + delay=30, + walk=1, + getblock=1, + blocktime=100) + +opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock') + +#-------------------------- Voltage Processing Unit ------------------------------------ + +procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId()) + +opObj00 = procUnitConfObj0.addOperation(name='selectChannels') +opObj00.addParameter(name='channelList', value='0,1,2,3,4', format='intlist') + +opObj01 = procUnitConfObj0.addOperation(name='setRadarFrequency') +opObj01.addParameter(name='frequency', value='30.e6', format='float') + +# opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights') +# opObj01.addParameter(name='topLim', value='73', format='int') +# opObj01.addParameter(name='botLim', value='71', format='int') + +opObj02 = procUnitConfObj0.addOperation(name='Decoder', optype='other') + +opObj03 = procUnitConfObj0.addOperation(name='CohInt', optype='other') +opObj03.addParameter(name='n', value='2', format='int') + +procUnitConfObj1 = controllerObj.addProcUnit(datatype='SpectraProc', inputId=procUnitConfObj0.getId()) +opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other') +opObj11.addParameter(name='id', value='237', format='int') +opObj11.addParameter(name='xmin', value='9.0', format='float') +opObj11.addParameter(name='xmax', value='16.0', format='float') +opObj11.addParameter(name='zmin', value='15.0', format='float') +opObj11.addParameter(name='zmax', value='50.0', format='float') + +#--------------------------- Parameters Processing Unit ------------------------------------ + +procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId()) +# +opObj10 = procUnitConfObj1.addOperation(name='SMDetection', optype='other') +opObj10.addParameter(name='azimuth', value='45', format='float') +opObj10.addParameter(name='hmin', value='60', format='float') +opObj10.addParameter(name='hmax', value='120', format='float') + +opObj12 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other') +opObj12.addParameter(name='path', value=pathfile) +opObj12.addParameter(name='blocksPerFile', value='1000', format='int') +opObj12.addParameter(name='metadataList',value='type,heightList,paramInterval,timeZone',format='list') +opObj12.addParameter(name='dataList',value='data_param,utctime',format='list') +opObj12.addParameter(name='mode',value='2',format='int') + +#-------------------------------------------------------------------------------------------------- + +controllerObj.start() + + diff --git a/schainpy/scripts/JASMET30_PhaseCal.py b/schainpy/scripts/JASMET30_PhaseCal.py new file mode 100644 index 0000000..1b0da1f --- /dev/null +++ b/schainpy/scripts/JASMET30_PhaseCal.py @@ -0,0 +1,68 @@ +import os, sys + +path = os.path.split(os.getcwd())[0] +path = os.path.split(path)[0] + +sys.path.insert(0, path) + +from schainpy.controller import Project + +controllerObj = Project() +controllerObj.setup(id = '004', name='script04', description="JASMET Phase Calibration") + +#-------------------------------------- Setup ----------------------------------------- +#Verificar estas variables + +#Path donde estan los archivos HDF5 de meteoros +path = os.path.join(os.environ['HOME'],'Pictures/JASMET30_mp/201608/meteor') + +#Path para los graficos +pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30_mp/201608/graphics') + +#Path donde se almacenaran las fases calculadas +pathfile = os.path.join(os.environ['HOME'],'Pictures/JASMET30_mp/201608/phase') + +#Fechas para busqueda de archivos +startDate = '2016/08/29' +endDate = '2016/09/11' +#Horas para busqueda de archivos +startTime = '00:00:00' +endTime = '23:59:59' + +#------------------------------------------------------------------------------------------------ +readUnitConfObj = controllerObj.addReadUnit(datatype='ParamReader', + path=path, + startDate=startDate, + endDate=endDate, + startTime=startTime, + endTime=endTime, + walk=1) + +#-------------------------------------------------------------------------------------------------- + +procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId()) +# # +opObj31 = procUnitConfObj1.addOperation(name='SMPhaseCalibration', optype='other') +opObj31.addParameter(name='nHours', value='1', format='float') +opObj31.addParameter(name='hmin', value='60', format='float') +opObj31.addParameter(name='hmax', value='120', format='float') +# opObj31.addParameter(name='channelPositions', value='(2.5,0),(0,2.5),(0,0),(0,4.5),(-2,0)', format='pairslist') + +opObj32 = procUnitConfObj1.addOperation(name='PhasePlot', optype='other') +opObj32.addParameter(name='id', value='201', format='int') +opObj32.addParameter(name='wintitle', value='PhaseCalibration', format='str') +opObj32.addParameter(name='save', value='1', format='bool') +opObj32.addParameter(name='xmin', value='0', format='float') +opObj32.addParameter(name='xmax', value='24', format='float') +opObj32.addParameter(name='ymin', value='-180', format='float') +opObj32.addParameter(name='ymax', value='180', format='float') +opObj32.addParameter(name='figpath', value=pathfig, format='str') +# # +opObj33 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other') +opObj33.addParameter(name='path', value=pathfile) +opObj33.addParameter(name='blocksPerFile', value='1000', format='int') +opObj33.addParameter(name='metadataList',value='type,outputInterval,timeZone',format='list') +opObj33.addParameter(name='dataList',value='data_output,utctime',format='list') +# # opObj25.addParameter(name='mode',value='1,0,0',format='intlist') + +controllerObj.start() \ No newline at end of file diff --git a/schainpy/scripts/JASMET30_Winds.py b/schainpy/scripts/JASMET30_Winds.py new file mode 100644 index 0000000..f489be5 --- /dev/null +++ b/schainpy/scripts/JASMET30_Winds.py @@ -0,0 +1,85 @@ +import os, sys + +path = os.path.split(os.getcwd())[0] +path = os.path.split(path)[0] + +sys.path.insert(0, path) + +from schainpy.controller import Project + +controllerObj = Project() +controllerObj.setup(id = '005', name='script05', description="JASMET Wind Estimation") + +#-------------------------------------- Setup ----------------------------------------- +#Verificar estas variables + +#Path donde estan los archivos HDF5 de meteoros +path = os.path.join(os.environ['HOME'],'Pictures/JASMET30_mp/201608/meteor') + +#Path para los graficos +pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30_mp/201608/graphics') + +#Path donde se almacenaran las estimaciones de vientos +pathfile = os.path.join(os.environ['HOME'],'Pictures/JASMET30_mp/201608/phase') + +#Fechas para busqueda de archivos +startDate = '2016/08/29' +endDate = '2016/09/11' +#Horas para busqueda de archivos +startTime = '00:00:00' +endTime = '23:59:59' + +#Offsets optimos obtenidos con OptimumOffset.py +phaseOffsets = '-2.84, -1.77, 11.94, 9.71' +phaseOffsets = '-5.86, -0.93, -7.29, 23.35' +#------------------------------------------------------------------------------------------------ +readUnitConfObj = controllerObj.addReadUnit(datatype='ParamReader', + path=path, + startDate=startDate, + endDate=endDate, + startTime=startTime, + endTime=endTime, + walk=1) +#-------------------------------------------------------------------------------------------------- + +procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId()) +opObj10 = procUnitConfObj1.addOperation(name='CorrectSMPhases',optype='other') +opObj10.addParameter(name='phaseOffsets', value=phaseOffsets, format='floatlist') + +opObj13 = procUnitConfObj1.addOperation(name='SkyMapPlot', optype='other') +opObj13.addParameter(name='id', value='1', format='int') +opObj13.addParameter(name='wintitle', value='Sky Map', format='str') +opObj13.addParameter(name='save', value='1', format='bool') +opObj13.addParameter(name='figpath', value=pathfig, format='str') +opObj13.addParameter(name='ftp', value='1', format='int') +opObj13.addParameter(name='exp_code', value='15', format='int') +opObj13.addParameter(name='sub_exp_code', value='1', format='int') +opObj13.addParameter(name='tmin', value='0', format='int') +opObj13.addParameter(name='tmax', value='24', format='int') + +opObj22 = procUnitConfObj1.addOperation(name='WindProfiler', optype='other') +opObj22.addParameter(name='technique', value='Meteors', format='str') +opObj22.addParameter(name='nHours', value='1', format='float') +opObj22.addParameter(name='hmin', value='70', format='float') +opObj22.addParameter(name='hmax', value='120', format='float') + +opObj23 = procUnitConfObj1.addOperation(name='WindProfilerPlot', optype='other') +opObj23.addParameter(name='id', value='2', format='int') +opObj23.addParameter(name='wintitle', value='Wind Profiler', format='str') +opObj23.addParameter(name='save', value='1', format='bool') +opObj23.addParameter(name='figpath', value = pathfig, format='str') +opObj23.addParameter(name='zmin', value='-140', format='int') +opObj23.addParameter(name='zmax', value='140', format='int') +opObj23.addParameter(name='xmin', value='0', format='float') +opObj23.addParameter(name='xmax', value='24', format='float') +opObj23.addParameter(name='ymin', value='70', format='float') +opObj23.addParameter(name='ymax', value='110', format='float') + +opObj33 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other') +opObj33.addParameter(name='path', value=pathfile) +opObj33.addParameter(name='blocksPerFile', value='1000', format='int') +opObj33.addParameter(name='metadataList',value='type,outputInterval,timeZone',format='list') +opObj33.addParameter(name='dataList',value='data_output,utctime',format='list') +#-------------------------------------------------------------------------------------------------- + +controllerObj.start() \ No newline at end of file diff --git a/schainpy/scripts/PPD.py b/schainpy/scripts/PPD.py new file mode 100644 index 0000000..0c66244 --- /dev/null +++ b/schainpy/scripts/PPD.py @@ -0,0 +1,97 @@ +import argparse + +from schainpy.controller import Project, multiSchain + +desc = "HF_EXAMPLE" + +def fiber(cursor, skip, q, dt): + + controllerObj = Project() + + controllerObj.setup(id='191', name='test01', description=desc) + + readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader', + path='/home/nanosat/data/sp1_f0', + startDate=dt, + endDate=dt, + startTime="00:00:00", + endTime="23:59:59", + online=0, + #set=1426485881, + walk=1, + queue=q, + cursor=cursor, + skip=skip, + verbose=1 + #timezone=-5*3600 + ) + + # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock') + # + procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId()) + # procUnitConfObj2.addParameter(name='nipp', value='5', format='int') + + procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId()) + opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other') + + # + # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other') + # opObj11.addParameter(name='id', value='1000', format='int') + # opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str') + # opObj11.addParameter(name='channelList', value='0', format='intlist') + # opObj11.addParameter(name='zmin', value='-120', format='float') + # opObj11.addParameter(name='zmax', value='-70', format='float') + # opObj11.addParameter(name='save', value='1', format='int') + # opObj11.addParameter(name='figpath', value=figpath, format='str') + + # opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other') + # opObj11.addParameter(name='channelList', value='0', format='intList') + + # opObj11.addParameter(name='id', value='2000', format='int') + # # opObj11.addParameter(name='colormap', value='0', format='bool') + # opObj11.addParameter(name='onlySNR', value='1', format='bool') + # opObj11.addParameter(name='DOP', value='0', format='bool') + # # opObj11.addParameter(name='showSNR', value='1', format='bool') + # # opObj11.addParameter(name='SNRthresh', value='0', format='int') + # opObj11.addParameter(name='SNRmin', value='-10', format='int') + # opObj11.addParameter(name='SNRmax', value='30', format='int') + + # opObj11.addParameter(name='showSNR', value='1', format='int') + # # opObj11.addParameter(name='channelList', value='0', format='intlist') + # # opObj11.addParameter(name='xmin', value='0', format='float') + # opObj11.addParameter(name='xmin', value='0', format='float') + # opObj11.addParameter(name='xmax', value='24', format='float') + + # opObj11.addParameter(name='zmin', value='-110', format='float') + # opObj11.addParameter(name='zmax', value='-70', format='float') + # opObj11.addParameter(name='save', value='0', format='int') + # # opObj11.addParameter(name='figpath', value='/tmp/', format='str') + # + opObj12 = procUnitConfObj3.addOperation(name='PublishData', optype='other') + opObj12.addParameter(name='zeromq', value=1, format='int') + opObj12.addParameter(name='verbose', value=0, format='bool') + + + # opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other') + # opObj13.addParameter(name='zeromq', value=1, format='int') + # opObj13.addParameter(name='server', value="juanca", format='str') + + opObj12.addParameter(name='delay', value=0, format='int') + + + # print "Escribiendo el archivo XML" + # controllerObj.writeXml(filename) + # print "Leyendo el archivo XML" + # controllerObj.readXml(filename) + + + # timeit.timeit('controllerObj.run()', number=2) + + controllerObj.start() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Set number of parallel processes') + parser.add_argument('--nProcess', default=1, type=int) + args = parser.parse_args() + multiSchain(fiber, nProcess=args.nProcess, startDate='2017/01/26', endDate='2017/01/26') diff --git a/schainpy/scripts/julia_mp.py b/schainpy/scripts/julia_mp.py new file mode 100644 index 0000000..2a8eaf1 --- /dev/null +++ b/schainpy/scripts/julia_mp.py @@ -0,0 +1,94 @@ +import argparse + +from schainpy.controller import Project, multiSchain + +desc = "HF_EXAMPLE" + +def fiber(cursor, skip, q, dt): + + controllerObj = Project() + + controllerObj.setup(id='191', name='test01', description=desc) + + readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader', + path='/home/nanosat/data/julia', + startDate=dt, + endDate=dt, + startTime="00:00:00", + endTime="23:59:59", + online=0, + #set=1426485881, + delay=10, + walk=1, + queue=q, + cursor=cursor, + skip=skip, + #timezone=-5*3600 + ) + + # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock') + # + procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId()) + # procUnitConfObj2.addParameter(name='nipp', value='5', format='int') + + # procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId()) + # opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other') + + # + opObj11 = procUnitConfObj2.addOperation(name='RTIPlot', optype='other') + opObj11.addParameter(name='id', value='1000', format='int') + opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str') + opObj11.addParameter(name='xmin', value='0', format='int') + opObj11.addParameter(name='xmax', value='24', format='int') + + # opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other') + # opObj11.addParameter(name='channelList', value='0', format='intList') + # + # opObj11.addParameter(name='id', value='2000', format='int') + # # opObj11.addParameter(name='colormap', value='0', format='bool') + # opObj11.addParameter(name='onlySNR', value='1', format='bool') + # opObj11.addParameter(name='DOP', value='0', format='bool') + # opObj11.addParameter(name='showSNR', value='1', format='bool') + # opObj11.addParameter(name='SNRthresh', value='0', format='int') + # opObj11.addParameter(name='SNRmin', value='-10', format='int') + # opObj11.addParameter(name='SNRmax', value='30', format='int') + + # opObj11.addParameter(name='showSNR', value='1', format='int') + # # opObj11.addParameter(name='channelList', value='0', format='intlist') + # # opObj11.addParameter(name='xmin', value='0', format='float') + # opObj11.addParameter(name='xmin', value='0', format='float') + # opObj11.addParameter(name='xmax', value='24', format='float') + + # opObj11.addParameter(name='zmin', value='-110', format='float') + # opObj11.addParameter(name='zmax', value='-70', format='float') + # opObj11.addParameter(name='save', value='0', format='int') + # # opObj11.addParameter(name='figpath', value='/tmp/', format='str') + # + # opObj12 = procUnitConfObj2.addOperation(name='PublishData', optype='other') + # opObj12.addParameter(name='zeromq', value=1, format='int') + # opObj12.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str') + + + # opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other') + # opObj13.addParameter(name='zeromq', value=1, format='int') + # opObj13.addParameter(name='server', value="juanca", format='str') + + # opObj12.addParameter(name='delay', value=1, format='int') + + + # print "Escribiendo el archivo XML" + # controllerObj.writeXml(filename) + # print "Leyendo el archivo XML" + # controllerObj.readXml(filename) + + + # timeit.timeit('controllerObj.run()', number=2) + + controllerObj.start() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Set number of parallel processes') + parser.add_argument('--nProcess', default=1, type=int) + args = parser.parse_args() + multiSchain(fiber, nProcess=args.nProcess, startDate='2016/08/19', endDate='2016/08/19') diff --git a/schainpy/scripts/optimun_offset.py b/schainpy/scripts/optimun_offset.py index 8ebcba0..fa6e660 100644 --- a/schainpy/scripts/optimun_offset.py +++ b/schainpy/scripts/optimun_offset.py @@ -93,7 +93,11 @@ def filterOffsets(offsets0, stdvLimit): #---------------------- Setup --------------------------- +<<<<<<< HEAD +path = '/home/nanosat/Pictures/JASMET30_mp/201608/phase' +======= path = '/home/jespinoza/Pictures/JASMET30/201608/phase' +>>>>>>> master stdvLimit = 0.5 #---------------------- Script --------------------------- diff --git a/schainpy/scripts/schain.xml b/schainpy/scripts/schain.xml new file mode 100644 index 0000000..04af99d --- /dev/null +++ b/schainpy/scripts/schain.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/schainpy/utils/log.py b/schainpy/utils/log.py index 85f1489..bbb532b 100644 --- a/schainpy/utils/log.py +++ b/schainpy/utils/log.py @@ -1,4 +1,4 @@ -""". +''' SCHAINPY - LOG Simple helper for log standarization Usage: @@ -13,33 +13,32 @@ SCHAINPY - LOG which will look like this: [NEVER GONNA] - give you up with color red as background and white as foreground. -""" +''' import click - -def warning(message): - click.echo(click.style('[WARNING] - ' + message, fg='yellow')) +def warning(message, tag='Warning'): + click.echo(click.style('[{}] {}'.format(tag, message), fg='yellow')) pass -def error(message): - click.echo(click.style('[ERROR] - ' + message, fg='red')) +def error(message, tag='Error'): + click.echo(click.style('[{}] {}'.format(tag, message), fg='red')) pass -def success(message): - click.echo(click.style(message, fg='green')) +def success(message, tag='Info'): + click.echo(click.style('[{}] {}'.format(tag, message), fg='green')) pass -def log(message): - click.echo('[LOG] - ' + message) +def log(message, tag='Info'): + click.echo('[{}] {}'.format(tag, message)) pass -def makelogger(topic, bg='reset', fg='reset'): +def makelogger(tag, bg='reset', fg='reset'): def func(message): - click.echo(click.style('[{}] - '.format(topic.upper()) + message, + click.echo(click.style('[{}] {}'.format(tag.upper(), message), bg=bg, fg=fg)) return func diff --git a/schainpy/utils/paramsFinder.py b/schainpy/utils/paramsFinder.py new file mode 100644 index 0000000..3361134 --- /dev/null +++ b/schainpy/utils/paramsFinder.py @@ -0,0 +1,81 @@ +import schainpy +from schainpy.model import Operation, ProcessingUnit +from importlib import import_module +from pydoc import locate + +def clean_modules(module): + noEndsUnder = [x for x in module if not x.endswith('__')] + noStartUnder = [x for x in noEndsUnder if not x.startswith('__')] + noFullUpper = [x for x in noStartUnder if not x.isupper()] + return noFullUpper + +def check_module(possible, instance): + def check(x): + try: + instancia = locate('schainpy.model.{}'.format(x)) + return isinstance(instancia(), instance) + except Exception as e: + return False + clean = clean_modules(possible) + return [x for x in clean if check(x)] + + +def getProcs(): + module = dir(import_module('schainpy.model')) + procs = check_module(module, ProcessingUnit) + try: + procs.remove('ProcessingUnit') + except Exception as e: + pass + return procs + +def getOperations(): + module = dir(import_module('schainpy.model')) + noProcs = [x for x in module if not x.endswith('Proc')] + operations = check_module(noProcs, Operation) + try: + operations.remove('Operation') + except Exception as e: + pass + return operations + +def getArgs(op): + module = locate('schainpy.model.{}'.format(op)) + args = module().getAllowedArgs() + try: + args.remove('self') + except Exception as e: + pass + try: + args.remove('dataOut') + except Exception as e: + pass + return args + +def getAll(): + allModules = dir(import_module('schainpy.model')) + modules = check_module(allModules, Operation) + modules.extend(check_module(allModules, ProcessingUnit)) + return modules + +def formatArgs(op): + args = getArgs(op) + + argsAsKey = ["\t'{}'".format(x) for x in args] + argsFormatted = ": 'string',\n".join(argsAsKey) + + print op + print "parameters = { \n" + argsFormatted + ": 'string',\n }" + print '\n' + + +if __name__ == "__main__": + getAll() + [formatArgs(x) for x in getAll()] + + ''' + parameters = { + 'id': , + 'wintitle': , + } + ''' \ No newline at end of file diff --git a/setup.py b/setup.py index 90fa3cf..be416d7 100644 --- a/setup.py +++ b/setup.py @@ -1,58 +1,69 @@ -""". - +''' Created on Jul 16, 2014 @author: Miguel Urco -""" +''' -import numpy from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext as _build_ext from schainpy import __version__ +class build_ext(_build_ext): + def finalize_options(self): + _build_ext.finalize_options(self) + # Prevent numpy from thinking it is still in its setup process: + __builtins__.__NUMPY_SETUP__ = False + import numpy + self.include_dirs.append(numpy.get_include()) + + setup(name="schainpy", version=__version__, description="Python tools to read, write and process Jicamarca data", author="Miguel Urco", author_email="miguel.urco@jro.igp.gob.pe", url="http://jro.igp.gob.pe", - packages={'schainpy', - 'schainpy.model', - 'schainpy.model.data', - 'schainpy.model.graphics', - 'schainpy.model.io', - 'schainpy.model.proc', - 'schainpy.model.serializer', - 'schainpy.model.utils', - 'schainpy.gui', - 'schainpy.gui.figures', - 'schainpy.gui.viewcontroller', - 'schainpy.gui.viewer', - 'schainpy.gui.viewer.windows'}, + packages = {'schainpy', + 'schainpy.model', + 'schainpy.model.data', + 'schainpy.model.graphics', + 'schainpy.model.io', + 'schainpy.model.proc', + 'schainpy.model.serializer', + 'schainpy.model.utils', + 'schainpy.gui', + 'schainpy.gui.figures', + 'schainpy.gui.viewcontroller', + 'schainpy.gui.viewer', + 'schainpy.gui.viewer.windows'}, ext_package='schainpy', py_modules=[''], package_data={'': ['schain.conf.template'], - 'schainpy.gui.figures': ['*.png', '*.jpg'], + 'schainpy.gui.figures': ['*.png','*.jpg'], }, include_package_data=False, + scripts =['schainpy/gui/schainGUI'], + ext_modules=[ + Extension("cSchain", ["schainpy/model/proc/extensions.c"] + )], entry_points={ 'console_scripts': [ 'schain = schaincli.cli:main', ], }, - scripts=['schainpy/gui/schainGUI'], - ext_modules=[Extension("cSchain", ["schainpy/model/proc/extensions.c"], include_dirs=[numpy.get_include()])], + cmdclass={'build_ext':build_ext}, + setup_requires=["numpy >= 1.11.2"], install_requires=[ - "scipy >= 0.13.0", - "h5py >= 2.2.1", - "matplotlib >= 1.4.2", - "pyfits >= 3.4", - "paramiko >= 2.1.2", - "paho-mqtt >= 1.2", - "zmq", - "fuzzywuzzy", - "click", - "colorama", - "python-Levenshtein" - ], - ) - + "scipy >= 0.14.0", + "h5py >= 2.2.1", + "matplotlib >= 1.4.2", + "pyfits >= 3.4", + "paramiko >= 2.1.2", + "paho-mqtt >= 1.2", + "zmq", + "fuzzywuzzy", + "click", + "colorama", + "python-Levenshtein" + ], + )