diff --git a/.gitignore b/.gitignore index 7784271..5294a6f 100644 --- a/.gitignore +++ b/.gitignore @@ -101,6 +101,9 @@ ENV/ .project .pydevproject <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> v2.3 # vscode @@ -108,6 +111,7 @@ ENV/ schaingui/node_modules/ schainpy/scripts/ +<<<<<<< HEAD ======= .svn/ *.png @@ -117,3 +121,12 @@ schainpy/scripts schaingui/node_modules trash >>>>>>> master +======= +schaingui/node_modules/ +.svn/ +*.png +*.pyc +*.xml +*.log +trash +>>>>>>> v2.3 diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 9b1c9b9..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "python.linting.pylintEnabled": true, - "git.ignoreLimitWarning": true -} \ No newline at end of file diff --git a/experiment_project.py b/experiment_project.py deleted file mode 100644 index 5bb268a..0000000 --- a/experiment_project.py +++ /dev/null @@ -1,34 +0,0 @@ -from schainpy.controller import Project - -desc = "A schain project" - -controller = Project() -controller.setup(id='191', name="project", description=desc) - -readUnitConf = controller.addReadUnit(datatype='VoltageReader', - path="/home/nanosat/schain", - startDate="1970/01/01", - endDate="2017/12/31", - startTime="00:00:00", - endTime="23:59:59", - online=0, - verbose=1, - walk=1, - ) - -procUnitConf1 = controller.addProcUnit(datatype='VoltageProc', inputId=readUnitConf.getId()) - -opObj11 = procUnitConf1.addOperation(name='ProfileSelector', optype='other') -opObj11.addParameter(name='profileRangeList', value='120,183', format='intlist') - -opObj11 = procUnitConf1.addOperation(name='RTIPlot', optype='other') -opObj11.addParameter(name='wintitle', value='Jicamarca Radio Observatory', format='str') -opObj11.addParameter(name='showprofile', value='0', format='int') -opObj11.addParameter(name='xmin', value='0', format='int') -opObj11.addParameter(name='xmax', value='24', format='int') -opObj11.addParameter(name='figpath', value="/home/nanosat/schain/figs", format='str') -opObj11.addParameter(name='wr_period', value='5', format='int') -opObj11.addParameter(name='exp_code', value='22', format='int') - - -controller.start() diff --git a/schain.xml b/schain.xml deleted file mode 100644 index a18a91f..0000000 --- a/schain.xml +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/schaincli/cli.py b/schaincli/cli.py index fdf7531..5b03de2 100644 --- a/schaincli/cli.py +++ b/schaincli/cli.py @@ -8,12 +8,13 @@ save_stdout = sys.stdout sys.stdout = open('trash', 'w') from multiprocessing import cpu_count from schaincli import templates -from schainpy import controller_api +from schainpy.controller import Project from schainpy.model import Operation, ProcessingUnit from schainpy.utils import log from importlib import import_module from pydoc import locate from fuzzywuzzy import process +from schainpy.utils import paramsFinder sys.stdout = save_stdout @@ -75,47 +76,25 @@ def search(nextcommand): if nextcommand is None: log.error('There is no Operation/ProcessingUnit to search') elif nextcommand == 'procs': - module = dir(import_module('schainpy.model')) - procs = check_module(module, ProcessingUnit) - try: - procs.remove('ProcessingUnit') - except Exception as e: - pass + procs = paramsFinder.getProcs() log.success('Current ProcessingUnits are:\n\033[1m{}\033[0m'.format('\n'.join(procs))) elif nextcommand == 'operations': - module = dir(import_module('schainpy.model')) - noProcs = [x for x in module if not x.endswith('Proc')] - operations = check_module(noProcs, Operation) - try: - operations.remove('Operation') - except Exception as e: - pass + operations = paramsFinder.getOperations() log.success('Current Operations are:\n\033[1m{}\033[0m'.format('\n'.join(operations))) else: try: - module = locate('schainpy.model.{}'.format(nextcommand)) - args = module().getAllowedArgs() + args = paramsFinder.getArgs(nextcommand) log.warning('Use this feature with caution. It may not return all the allowed arguments') - try: - args.remove('self') - except Exception as e: - pass - try: - args.remove('dataOut') - except Exception as e: - pass if len(args) == 0: log.success('{} has no arguments'.format(nextcommand)) else: log.success('Showing arguments of {} are:\n\033[1m{}\033[0m'.format(nextcommand, '\n'.join(args))) except Exception as e: log.error('Module {} does not exists'.format(nextcommand)) - allModules = dir(import_module('schainpy.model')) - module = check_module(allModules, Operation) - module.extend(check_module(allModules, ProcessingUnit)) - similar = process.extractOne(nextcommand, module)[0] - log.success('Searching {} instead'.format(similar)) + allModules = paramsFinder.getAll() + similar = process.extractOne(nextcommand, allModules)[0] + log.success('Showing {} instead'.format(similar)) search(similar) @@ -171,18 +150,8 @@ def test(): def runFromXML(filename): - controller = controller_api.ControllerThread() + controller = Project() if not controller.readXml(filename): return - - plotterObj = controller.useExternalPlotter() - controller.start() - plotterObj.start() - - cliLogger("Finishing all processes") - - controller.join(5) - - cliLogger("End of script") return diff --git a/schainpy/CHANGELOG.md b/schainpy/CHANGELOG.md index 1ef44ce..1e83b22 100644 --- a/schainpy/CHANGELOG.md +++ b/schainpy/CHANGELOG.md @@ -5,16 +5,17 @@ * Added two new Processing Units `PublishData` and `ReceiverData` for receiving and sending dataOut through multiple ways (tcp, ipc, inproc). * Added a new graphics Processing Unit `PlotterReceiver`. It is decoupled from normal processing sequence with support for data generated by multiprocessing scripts. * Added support for sending realtime graphic to web server. -* XML command `schain` is now `schain --xml`. +* GUI command `schain` is now `schainGUI`. * Added a CLI tool named `schain`. * Scripts templates can be now generated with `schain generate`. * Now it is possible to search Processing Units and Operations with `schain search [module]` to get the right name and its allowed parameters. + * `schain xml` to run xml scripts. * Added suggestions when parameters are poorly written. * `Controller.start()` now runs in a different process than the process calling it. * Added `schainpy.utils.log` for log standarization. * Running script on online mode no longer ignores date and hour. Issue #1109. * Added support for receving voltage data directly from JARS (tcp, ipc). -* Updated README for compatibility with MAC OS GUI installation. +* Updated README for MAC OS GUI installation. * Setup now installs numpy. ### 2.2.6 diff --git a/schainpy/VERSION b/schainpy/VERSION deleted file mode 100644 index a19b1a9..0000000 --- a/schainpy/VERSION +++ /dev/null @@ -1,85 +0,0 @@ -VERSIONS: - -2.1.2: --jroutils_ftp.py: Bug fixed, Any error sending file stopped the Server Thread - Server thread opens and closes remote server each time file list is sent --jroplot_spectra.py: Noise path was not being created when noise data is saved. --jroIO_base.py: startTime can be greater than endTime. Example: SpreadF [18:00 - 07:00] - -2.1.3: --jroplot_heispectra.py: SpectraHeisScope was not showing the right channels --jroproc_voltage.py: Bug fixed selecting profiles (self.nProfiles took a wrong value), - Bug fixed selecting heights by block (selecting profiles instead heights) --jroproc_voltage.py: New feature added: decoding data by block using FFT. --jroIO_heispectra.py: Bug fixed in FitsReader. Using local Fits instance instead schainpy.mode.data.jrodata.Fits. --jroIO_heispectra.py: Channel index list does not exist. - -2.1.3.1: --GUI: every icon were resized --jroproc_voltage.py: Print a message when "Read from code" option is selected and the code is not defined inside data file - -2.1.3.2: --GUI: user interaction enhanced --controller_api.py: Safe access to ControllerThead - -2.1.3.3: --Colored Button Icons were added to GUI - -2.1.4: --Sending error notifications to signal chain administrator --Login to email server added - -2.1.4.1: --Send notifications when an error different to ValueError is detected - -2.1.4.2: --A new Plotter Class was added --Project.start() does not accept filename as a parameter anymore - -2.1.5: --serializer module added to Signal Chain --jroplotter.py added to Signal Chain - -2.2.0: --GUI: use of external plotter --Compatible with matplotlib 1.5.0 - -2.2.1: --Bugs fixed in GUI --Views were improved in GUI --Support to MST-ISR experiments --Bug fixed getting noise using hyldebrant. (minimum number of points > 20%) --handleError added to jroplotter.py - -2.2.2: --VoltageProc: ProfileSelector, Reshape, Decoder with nTxs!=1 and getblock=True was tested --Rawdata and testRawdata.py added to Signal Chain project - -2.2.3: --Bug fixed in GUI: Error getting(reading) Code value --Bug fixed in GUI: Flip option always needs channelList field --Bug fixed in jrodata: when one branch modified a value in "dataOut" (example: dataOut.code) this value -was modified for every branch (because this was a reference). It was modified in data.copy() --Bug fixed in jroproc_voltage.profileSelector(): rangeList replaces to profileRangeList. - - -2.2.3.1: --Filtering block by time has been added. --Bug fixed plotting RTI, CoherenceMap and others using xmin and xmax parameters. The first day worked -properly but the next days did not. - -2.2.4: --jroproc_spectra_lags.py added to schainpy --Bug fixed in schainGUI: ProcUnit was created with the same id in some cases. --Bug fixed in jroHeaderIO: Header size validation. - -2.2.4.1: --jroIO_usrp.py is update to read Sandra's data --decimation in Spectra and RTI plots is always enabled. --time-window option added to GUI - -2.2.5: --splitProfiles and combineProfiles modules were added to VoltageProc and Signal Chain GUI. --nProfiles of USRP data (hdf5) is the number of profiles thera are in one second. --jroPlotter works directly with data objects instead of dictionaries --script "schain" was added to Signal Chain installer \ No newline at end of file diff --git a/schainpy/__init__.py b/schainpy/__init__.py index af29ee4..d46a2d0 100644 --- a/schainpy/__init__.py +++ b/schainpy/__init__.py @@ -4,5 +4,4 @@ Created on Feb 7, 2012 @author $Author$ @version $Id$ ''' - __version__ = "2.3" diff --git a/schainpy/controller.py b/schainpy/controller.py index e400096..1b79f87 100644 --- a/schainpy/controller.py +++ b/schainpy/controller.py @@ -9,6 +9,7 @@ import datetime import traceback import schainpy import schainpy.admin +from schainpy.utils.log import logToFile from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring from xml.dom import minidom @@ -16,6 +17,8 @@ from xml.dom import minidom from schainpy.model import * from time import sleep + + def prettify(elem): """Return a pretty-printed XML string for the Element. """ @@ -23,6 +26,52 @@ def prettify(elem): reparsed = minidom.parseString(rough_string) return reparsed.toprettyxml(indent=" ") +def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False): + skip = 0 + cursor = 0 + nFiles = None + processes = [] + dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d') + dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d') + days = (dt2 - dt1).days + + for day in range(days+1): + skip = 0 + cursor = 0 + q = Queue() + processes = [] + dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d') + firstProcess = Process(target=child, args=(cursor, skip, q, dt)) + firstProcess.start() + if by_day: + continue + nFiles = q.get() + if nFiles==0: + continue + firstProcess.terminate() + skip = int(math.ceil(nFiles/nProcess)) + while True: + processes.append(Process(target=child, args=(cursor, skip, q, dt))) + processes[cursor].start() + if nFiles < cursor*skip: + break + cursor += 1 + + def beforeExit(exctype, value, trace): + for process in processes: + process.terminate() + process.join() + print traceback.print_tb(trace) + + sys.excepthook = beforeExit + + for process in processes: + process.join() + process.terminate() + + time.sleep(3) + + class ParameterConf(): id = None @@ -207,8 +256,8 @@ class ParameterConf(): print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format) -class OperationConf(): - +class OperationConf(): + id = None name = None priority = None @@ -623,8 +672,7 @@ class ProcUnitConf(): #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id) sts = self.procUnitObj.call(opType = opConfObj.type, opName = opConfObj.name, - opId = opConfObj.id, - **kwargs) + opId = opConfObj.id) # total_time = time.time() - ini # @@ -673,9 +721,9 @@ class ReadUnitConf(ProcUnitConf): def getElementName(self): return self.ELEMENTNAME - - def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, **kwargs): + def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="", + endTime="", parentId=None, queue=None, server=None, **kwargs): #Compatible with old signal chain version if datatype==None and name==None: raise ValueError, "datatype or name should be defined" @@ -811,9 +859,8 @@ class ReadUnitConf(ProcUnitConf): self.endDate = opConfObj.getParameterValue('endDate') self.startTime = opConfObj.getParameterValue('startTime') self.endTime = opConfObj.getParameterValue('endTime') - -class Project(): - + +class Project(Process): id = None name = None description = None @@ -824,13 +871,14 @@ class Project(): ELEMENTNAME = 'Project' plotterQueue = None - - def __init__(self, plotter_queue=None): - + + def __init__(self, plotter_queue=None, logfile=None): + Process.__init__(self) self.id = None self.name = None self.description = None - + if logfile is not None: + logToFile(logfile) self.plotterQueue = plotter_queue self.procUnitConfObjDict = {} @@ -892,7 +940,6 @@ class Project(): self.description = description def addReadUnit(self, id=None, datatype=None, name=None, **kwargs): - if id is None: idReadUnit = self.__getNewId() else: @@ -1055,9 +1102,9 @@ class Project(): def printattr(self): print "Project[%s]: name = %s, description = %s" %(self.id, - self.name, - self.description) - + self.name, + self.description) + for procUnitConfObj in self.procUnitConfObjDict.values(): procUnitConfObj.printattr() @@ -1176,9 +1223,14 @@ class Project(): def useExternalPlotter(self): raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class" - - def run(self): + + + def run(self, filename=None): + # self.writeXml(filename) + self.createObjects() + self.connectObjects() + print print "*"*60 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__ @@ -1226,69 +1278,3 @@ class Project(): for procKey in keyList: procUnitConfObj = self.procUnitConfObjDict[procKey] procUnitConfObj.close() - - print "Process finished" - - def start(self): - - self.writeXml() - - self.createObjects() - self.connectObjects() - self.run() - -if __name__ == '__main__': - - desc = "Segundo Test" - filename = "schain.xml" - - controllerObj = Project() - - controllerObj.setup(id = '191', name='test01', description=desc) - - readUnitConfObj = controllerObj.addReadUnit(datatype='Voltage', - path='data/rawdata/', - startDate='2011/01/01', - endDate='2012/12/31', - startTime='00:00:00', - endTime='23:59:59', - online=1, - walk=1) - - procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage', inputId=readUnitConfObj.getId()) - - opObj10 = procUnitConfObj0.addOperation(name='selectChannels') - opObj10.addParameter(name='channelList', value='3,4,5', format='intlist') - - opObj10 = procUnitConfObj0.addOperation(name='selectHeights') - opObj10.addParameter(name='minHei', value='90', format='float') - opObj10.addParameter(name='maxHei', value='180', format='float') - - opObj12 = procUnitConfObj0.addOperation(name='CohInt', optype='external') - opObj12.addParameter(name='n', value='10', format='int') - - procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj0.getId()) - procUnitConfObj1.addParameter(name='nFFTPoints', value='32', format='int') -# procUnitConfObj1.addParameter(name='pairList', value='(0,1),(0,2),(1,2)', format='') - - - opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external') - opObj11.addParameter(name='idfigure', value='1', format='int') - opObj11.addParameter(name='wintitle', value='SpectraPlot0', format='str') - opObj11.addParameter(name='zmin', value='40', format='int') - opObj11.addParameter(name='zmax', value='90', format='int') - opObj11.addParameter(name='showprofile', value='1', format='int') - - print "Escribiendo el archivo XML" - - controllerObj.writeXml(filename) - - print "Leyendo el archivo XML" - controllerObj.readXml(filename) - #controllerObj.printattr() - - controllerObj.createObjects() - controllerObj.connectObjects() - controllerObj.run() - - \ No newline at end of file diff --git a/schainpy/model/__init__.py b/schainpy/model/__init__.py index 6dea354..3d76a6d 100644 --- a/schainpy/model/__init__.py +++ b/schainpy/model/__init__.py @@ -9,4 +9,4 @@ from data import * from io import * from proc import * from graphics import * -from utils import * +from utils import * \ No newline at end of file diff --git a/schainpy/model/data/jroheaderIO.py b/schainpy/model/data/jroheaderIO.py index 77324a3..c85ae5d 100644 --- a/schainpy/model/data/jroheaderIO.py +++ b/schainpy/model/data/jroheaderIO.py @@ -401,9 +401,9 @@ class RadarControllerHeader(Header): try: if hasattr(fp, 'read'): - self.nCode = numpy.fromfile(fp, '= self.data['throttle']: print 'passed', seconds_passed self.__plot() @@ -230,7 +316,6 @@ class PlotSpectraData(PlotData): z = self.data[self.CODE] for n, ax in enumerate(self.axes): - if ax.firsttime: self.xmax = self.xmax if self.xmax else np.nanmax(x) self.xmin = self.xmin if self.xmin else -self.xmax @@ -462,81 +547,184 @@ class PlotRTIData(PlotData): self.ncols = 1 self.nrows = self.dataOut.nChannels self.width = 10 - self.height = 2.2*self.nrows if self.nrows<6 else 12 + #TODO : arreglar la altura de la figura, esta hardcodeada. + #Se arreglo, testear! + if self.ind_plt_ch: + self.height = 3.2#*self.nrows if self.nrows<6 else 12 + else: + self.height = 2.2*self.nrows if self.nrows<6 else 12 + + ''' if self.nrows==1: self.height += 1 + ''' self.ylabel = 'Range [Km]' self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList] - if self.figure is None: - self.figure = plt.figure(figsize=(self.width, self.height), - edgecolor='k', - facecolor='w') - else: - self.figure.clf() - self.axes = [] - - for n in range(self.nrows): - ax = self.figure.add_subplot(self.nrows, self.ncols, n+1) - ax.firsttime = True - self.axes.append(ax) - - def plot(self): - - self.x = np.array(self.times) - self.y = self.dataOut.getHeiRange() - self.z = [] - - for ch in range(self.nrows): - self.z.append([self.data[self.CODE][t][ch] for t in self.times]) + ''' + Logica: + 1) Si la variable ind_plt_ch es True, va a crear mas de 1 figura + 2) guardamos "Figures" en una lista y "axes" en otra, quizas se deberia guardar el + axis dentro de "Figures" como un diccionario. + ''' + if self.ind_plt_ch is False: #standard mode + + if self.figure is None: #solo para la priemra vez + self.figure = plt.figure(figsize=(self.width, self.height), + edgecolor='k', + facecolor='w') + else: + self.figure.clf() + self.axes = [] - self.z = np.array(self.z) - for n, ax in enumerate(self.axes): - x, y, z = self.fill_gaps(*self.decimate()) - xmin = self.min_time - xmax = xmin+self.xrange*60*60 - self.zmin = self.zmin if self.zmin else np.min(self.z) - self.zmax = self.zmax if self.zmax else np.max(self.z) - if ax.firsttime: - self.ymin = self.ymin if self.ymin else np.nanmin(self.y) - self.ymax = self.ymax if self.ymax else np.nanmax(self.y) - plot = ax.pcolormesh(x, y, z[n].T, - vmin=self.zmin, - vmax=self.zmax, - cmap=plt.get_cmap(self.colormap) - ) - divider = make_axes_locatable(ax) - cax = divider.new_horizontal(size='2%', pad=0.05) - self.figure.add_axes(cax) - plt.colorbar(plot, cax) - ax.set_ylim(self.ymin, self.ymax) - ax.xaxis.set_major_formatter(FuncFormatter(func)) - ax.xaxis.set_major_locator(LinearLocator(6)) + for n in range(self.nrows): + ax = self.figure.add_subplot(self.nrows, self.ncols, n+1) + #ax = self.figure(n+1) + ax.firsttime = True + self.axes.append(ax) - ax.set_ylabel(self.ylabel) + else : #append one figure foreach channel in channelList + if self.figurelist == None: + self.figurelist = [] + for n in range(self.nrows): + self.figure = plt.figure(figsize=(self.width, self.height), + edgecolor='k', + facecolor='w') + #add always one subplot + self.figurelist.append(self.figure) + + else : # cada dia nuevo limpia el axes, pero mantiene el figure + for eachfigure in self.figurelist: + eachfigure.clf() # eliminaria todas las figuras de la lista? + self.axes = [] + + for eachfigure in self.figurelist: + ax = eachfigure.add_subplot(1,1,1) #solo 1 axis por figura + #ax = self.figure(n+1) + ax.firsttime = True + #Cada figura tiene un distinto puntero + self.axes.append(ax) + #plt.close(eachfigure) - # if self.xmin is None: - # xmin = self.min_time - # else: - # xmin = (datetime.datetime.combine(self.dataOut.datatime.date(), - # datetime.time(self.xmin, 0, 0))-d1970).total_seconds() - ax.set_xlim(xmin, xmax) - ax.firsttime = False - else: - ax.collections.remove(ax.collections[0]) - ax.set_xlim(xmin, xmax) - plot = ax.pcolormesh(x, y, z[n].T, - vmin=self.zmin, - vmax=self.zmax, - cmap=plt.get_cmap(self.colormap) - ) - ax.set_title('{} {}'.format(self.titles[n], - datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')), - size=8) + def plot(self): - self.saveTime = self.min_time + if self.ind_plt_ch is False: #standard mode + self.x = np.array(self.times) + self.y = self.dataOut.getHeiRange() + self.z = [] + + for ch in range(self.nrows): + self.z.append([self.data[self.CODE][t][ch] for t in self.times]) + + self.z = np.array(self.z) + for n, ax in enumerate(self.axes): + x, y, z = self.fill_gaps(*self.decimate()) + if self.xmin is None: + xmin = self.min_time + else: + xmin = fromtimestamp(int(self.xmin), self.min_time) + if self.xmax is None: + xmax = xmin + self.xrange*60*60 + else: + xmax = xmin + (self.xmax - self.xmin) * 60 * 60 + self.zmin = self.zmin if self.zmin else np.min(self.z) + self.zmax = self.zmax if self.zmax else np.max(self.z) + if ax.firsttime: + self.ymin = self.ymin if self.ymin else np.nanmin(self.y) + self.ymax = self.ymax if self.ymax else np.nanmax(self.y) + plot = ax.pcolormesh(x, y, z[n].T, + vmin=self.zmin, + vmax=self.zmax, + cmap=plt.get_cmap(self.colormap) + ) + divider = make_axes_locatable(ax) + cax = divider.new_horizontal(size='2%', pad=0.05) + self.figure.add_axes(cax) + plt.colorbar(plot, cax) + ax.set_ylim(self.ymin, self.ymax) + ax.xaxis.set_major_formatter(FuncFormatter(func)) + ax.xaxis.set_major_locator(LinearLocator(6)) + ax.set_ylabel(self.ylabel) + # if self.xmin is None: + # xmin = self.min_time + # else: + # xmin = (datetime.datetime.combine(self.dataOut.datatime.date(), + # datetime.time(self.xmin, 0, 0))-d1970).total_seconds() + + ax.set_xlim(xmin, xmax) + ax.firsttime = False + else: + ax.collections.remove(ax.collections[0]) + ax.set_xlim(xmin, xmax) + plot = ax.pcolormesh(x, y, z[n].T, + vmin=self.zmin, + vmax=self.zmax, + cmap=plt.get_cmap(self.colormap) + ) + ax.set_title('{} {}'.format(self.titles[n], + datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')), + size=8) + + self.saveTime = self.min_time + else : + self.x = np.array(self.times) + self.y = self.dataOut.getHeiRange() + self.z = [] + + for ch in range(self.nrows): + self.z.append([self.data[self.CODE][t][ch] for t in self.times]) + + self.z = np.array(self.z) + for n, eachfigure in enumerate(self.figurelist): #estaba ax in axes + + x, y, z = self.fill_gaps(*self.decimate()) + xmin = self.min_time + xmax = xmin+self.xrange*60*60 + self.zmin = self.zmin if self.zmin else np.min(self.z) + self.zmax = self.zmax if self.zmax else np.max(self.z) + if self.axes[n].firsttime: + self.ymin = self.ymin if self.ymin else np.nanmin(self.y) + self.ymax = self.ymax if self.ymax else np.nanmax(self.y) + plot = self.axes[n].pcolormesh(x, y, z[n].T, + vmin=self.zmin, + vmax=self.zmax, + cmap=plt.get_cmap(self.colormap) + ) + divider = make_axes_locatable(self.axes[n]) + cax = divider.new_horizontal(size='2%', pad=0.05) + eachfigure.add_axes(cax) + #self.figure2.add_axes(cax) + plt.colorbar(plot, cax) + self.axes[n].set_ylim(self.ymin, self.ymax) + + self.axes[n].xaxis.set_major_formatter(FuncFormatter(func)) + self.axes[n].xaxis.set_major_locator(LinearLocator(6)) + + self.axes[n].set_ylabel(self.ylabel) + + if self.xmin is None: + xmin = self.min_time + else: + xmin = (datetime.datetime.combine(self.dataOut.datatime.date(), + datetime.time(self.xmin, 0, 0))-d1970).total_seconds() + + self.axes[n].set_xlim(xmin, xmax) + self.axes[n].firsttime = False + else: + self.axes[n].collections.remove(self.axes[n].collections[0]) + self.axes[n].set_xlim(xmin, xmax) + plot = self.axes[n].pcolormesh(x, y, z[n].T, + vmin=self.zmin, + vmax=self.zmax, + cmap=plt.get_cmap(self.colormap) + ) + self.axes[n].set_title('{} {}'.format(self.titles[n], + datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')), + size=8) + + self.saveTime = self.min_time class PlotCOHData(PlotRTIData): @@ -549,6 +737,7 @@ class PlotCOHData(PlotRTIData): self.nrows = self.dataOut.nPairs self.width = 10 self.height = 2.2*self.nrows if self.nrows<6 else 12 + self.ind_plt_ch = False #just for coherence and phase if self.nrows==1: self.height += 1 self.ylabel = 'Range [Km]' diff --git a/schainpy/model/graphics/jroplot_parameters.py b/schainpy/model/graphics/jroplot_parameters.py index dd1dbb1..ffc1924 100644 --- a/schainpy/model/graphics/jroplot_parameters.py +++ b/schainpy/model/graphics/jroplot_parameters.py @@ -14,7 +14,6 @@ class MomentsPlot(Figure): WIDTHPROF = None HEIGHTPROF = None PREFIX = 'prm' - def __init__(self, **kwargs): Figure.__init__(self, **kwargs) self.isConfig = False @@ -1417,6 +1416,7 @@ class PhasePlot(Figure): PREFIX = 'mphase' + def __init__(self, **kwargs): Figure.__init__(self, **kwargs) self.timerange = 24*60*60 diff --git a/schainpy/model/graphics/jroplot_spectra.py b/schainpy/model/graphics/jroplot_spectra.py index 815ce1b..d2b11f1 100644 --- a/schainpy/model/graphics/jroplot_spectra.py +++ b/schainpy/model/graphics/jroplot_spectra.py @@ -10,6 +10,7 @@ import numpy from figure import Figure, isRealtime, isTimeInHourRange from plotting_codes import * + class SpectraPlot(Figure): isConfig = None diff --git a/schainpy/model/io/jroIO_base.py b/schainpy/model/io/jroIO_base.py index d7a11ee..4824af8 100644 --- a/schainpy/model/io/jroIO_base.py +++ b/schainpy/model/io/jroIO_base.py @@ -541,8 +541,8 @@ class JRODataIO: return inspect.getargspec(self.run).args class JRODataReader(JRODataIO): - - + + firstTime = True online = 0 realtime = 0 @@ -578,8 +578,8 @@ class JRODataReader(JRODataIO): selBlocksize = None selBlocktime = None - - + + onlineWithDate = False def __init__(self): """ @@ -616,7 +616,6 @@ class JRODataReader(JRODataIO): cursor=None, skip=None, walk=True): - self.filenameList = [] self.datetimeList = [] @@ -682,34 +681,35 @@ class JRODataReader(JRODataIO): print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) self.filenameList = filenameList - self.datetimeList = datetimeList - + self.datetimeList = datetimeList return pathList, filenameList - def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None): - + def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None): + """ - Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y - devuelve el archivo encontrado ademas de otros datos. - - Input: - path : carpeta donde estan contenidos los files que contiene data - - expLabel : Nombre del subexperimento (subfolder) - - ext : extension de los files - - walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath) - - Return: - directory : eL directorio donde esta el file encontrado - filename : el ultimo file de una determinada carpeta - year : el anho - doy : el numero de dia del anho - set : el set del archivo - - + Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y + devuelve el archivo encontrado ademas de otros datos. + + Input: + path : carpeta donde estan contenidos los files que contiene data + + expLabel : Nombre del subexperimento (subfolder) + + ext : extension de los files + + walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath) + + Return: + directory : eL directorio donde esta el file encontrado + filename : el ultimo file de una determinada carpeta + year : el anho + doy : el numero de dia del anho + set : el set del archivo + + """ + pathList = None + filenameList = None if not os.path.isdir(path): return None, None, None, None, None, None @@ -719,7 +719,7 @@ class JRODataReader(JRODataIO): fullpath = path foldercounter = 0 else: - #Filtra solo los directorios + # Filtra solo los directorios for thisPath in os.listdir(path): if not os.path.isdir(os.path.join(path,thisPath)): continue @@ -755,7 +755,7 @@ class JRODataReader(JRODataIO): year = int( filename[1:5] ) doy = int( filename[5:8] ) - set = int( filename[8:11] ) + set = int( filename[8:11] ) return fullpath, foldercounter, filename, year, doy, set @@ -767,7 +767,7 @@ class JRODataReader(JRODataIO): idFile += 1 if not(idFile < len(self.filenameList)): self.flagNoMoreFiles = 1 - # print "[Reading] No more Files" + # print "[Reading] No more Files" return 0 filename = self.filenameList[idFile] @@ -785,31 +785,32 @@ class JRODataReader(JRODataIO): self.fileSize = fileSize self.fp = fp - # print "[Reading] Setting the file: %s"%self.filename + #print "[Reading] Setting the file: %s"%self.filename return 1 def __setNextFileOnline(self): """ - Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si - no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files - siguientes. - - Affected: - self.flagIsNewFile - self.filename - self.fileSize - self.fp - self.set - self.flagNoMoreFiles - - Return: - 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado - 1 : si el file fue abierto con exito y esta listo a ser leido - - Excepciones: - Si un determinado file no puede ser abierto - """ + Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si + no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files + siguientes. + + Affected: + self.flagIsNewFile + self.filename + self.fileSize + self.fp + self.set + self.flagNoMoreFiles + + Return: + 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado + 1 : si el file fue abierto con exito y esta listo a ser leido + + Excepciones: + Si un determinado file no puede ser abierto + """ + nFiles = 0 fileOk_flag = False firstTime_flag = True @@ -882,13 +883,34 @@ class JRODataReader(JRODataIO): def setNextFile(self): if self.fp != None: self.fp.close() - if self.online: newFile = self.__setNextFileOnline() else: newFile = self.__setNextFileOffline() - if not(newFile): + if self.onlineWithDate is True: + self.onlineWithDate=False + self.online = True + self.firstTime = False + self.setup( + path=self.path, + startDate=self.startDate, + endDate=self.endDate, + startTime=self.startTime , + endTime=self.endTime, + set=self.set, + expLabel=self.expLabel, + ext=self.ext, + online=self.online, + delay=self.delay, + walk=self.walk, + getblock=self.getblock, + nTxs=self.nTxs, + realtime=self.realtime, + blocksize=self.blocksize, + blocktime=self.blocktime + ) + return 1 print '[Reading] No more files to read' return 0 @@ -1010,7 +1032,7 @@ class JRODataReader(JRODataIO): if not(self.setNextFile()): return 0 else: - return 1 + return 1 #if self.server is None: currentSize = self.fileSize - self.fp.tell() neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize @@ -1042,15 +1064,13 @@ class JRODataReader(JRODataIO): def readNextBlock(self): #Skip block out of startTime and endTime - while True: + while True: if not(self.__setNewBlock()): print 'returning' return 0 if not(self.readBlock()): return 0 - self.getBasicHeader() - if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime): print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks, @@ -1062,8 +1082,8 @@ class JRODataReader(JRODataIO): if self.verbose: print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks, - self.processingHeaderObj.dataBlocksPerFile, - self.dataOut.datatime.ctime()) + self.processingHeaderObj.dataBlocksPerFile, + self.dataOut.datatime.ctime()) return 1 def __readFirstHeader(self): @@ -1257,13 +1277,13 @@ class JRODataReader(JRODataIO): def setup(self, path=None, - startDate=None, - endDate=None, - startTime=datetime.time(0,0,0), - endTime=datetime.time(23,59,59), - set=None, - expLabel = "", - ext = None, + startDate=None, + endDate=None, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + set=None, + expLabel = "", + ext = None, online = False, delay = 60, walk = True, @@ -1272,80 +1292,137 @@ class JRODataReader(JRODataIO): realtime=False, blocksize=None, blocktime=None, - queue=None, - skip=None, - cursor=None, - warnings=True, verbose=True, - server=None): - if server is not None: - if 'tcp://' in server: - address = server - else: - address = 'ipc:///tmp/%s' % server - self.server = address - self.context = zmq.Context() - self.receiver = self.context.socket(zmq.PULL) - self.receiver.connect(self.server) - time.sleep(0.5) - print '[Starting] ReceiverData from {}'.format(self.server) - else: - self.server = None - if path == None: - raise ValueError, "[Reading] The path is not valid" - - if ext == None: - ext = self.ext - - if online: - print "[Reading] Searching files in online mode..." + **kwargs): - for nTries in range( self.nTries ): - fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set) - - if fullpath: - break - - print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1) - sleep( self.delay ) - - if not(fullpath): - print "[Reading] There 'isn't any valid file in %s" % path - return + if path == None: + raise ValueError, "[Reading] The path is not valid" + - self.year = year - self.doy = doy - self.set = set - 1 - self.path = path - self.foldercounter = foldercounter - last_set = None - else: - print "[Reading] Searching files in offline mode ..." - pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate, + if ext == None: + ext = self.ext + + self.verbose=verbose + self.path = path + self.startDate = startDate + self.endDate = endDate + self.startTime = startTime + self.endTime = endTime + self.set = set + self.expLabel = expLabel + self.ext = ext + self.online = online + self.delay = delay + self.walk = walk + self.getblock = getblock + self.nTxs = nTxs + self.realtime = realtime + self.blocksize = blocksize + self.blocktime = blocktime + + + if self.firstTime is True: + pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate, startTime=startTime, endTime=endTime, set=set, expLabel=expLabel, ext=ext, - walk=walk, cursor=cursor, - skip=skip, queue=queue) - - if not(pathList): - # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path, - # datetime.datetime.combine(startDate,startTime).ctime(), - # datetime.datetime.combine(endDate,endTime).ctime()) - - # sys.exit(-1) + walk=walk) + if filenameList is not None: filenameList = filenameList[:-1] + if pathList is not None and filenameList is not None and online: + self.onlineWithDate = True + online = False self.fileIndex = -1 - self.pathList = [] - self.filenameList = [] - return + self.pathList = pathList + self.filenameList = filenameList + file_name = os.path.basename(filenameList[-1]) + basename, ext = os.path.splitext(file_name) + last_set = int(basename[-3:]) + + if online: + print "[Reading] Searching files in online mode..." + + for nTries in range(self.nTries): + fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, + expLabel=expLabel, + ext=ext, + walk=walk, + startDate=startDate, + startTime=startTime, + set=set) + + if fullpath: + break + print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1) + sleep( self.delay ) + + if not(fullpath): + print "[Reading] There 'isn't any valid file in %s" % path + return + + self.year = year + self.doy = doy + self.set = set - 1 + self.path = path + self.foldercounter = foldercounter + last_set = None + else: + print "[Reading] Searching files in offline mode ..." + pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate, + startTime=startTime, endTime=endTime, + set=set, expLabel=expLabel, ext=ext, + walk=walk) + + if not(pathList): + # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path, + # datetime.datetime.combine(startDate,startTime).ctime(), + # datetime.datetime.combine(endDate,endTime).ctime()) + + # sys.exit(-1) + + self.fileIndex = -1 + self.pathList = [] + self.filenameList = [] + return + + self.fileIndex = -1 + self.pathList = pathList + self.filenameList = filenameList + file_name = os.path.basename(filenameList[-1]) + basename, ext = os.path.splitext(file_name) + last_set = int(basename[-3:]) + + self.online = online + self.realtime = realtime + self.delay = delay + ext = ext.lower() + self.ext = ext + self.getByBlock = getblock + self.nTxs = nTxs + self.startTime = startTime + self.endTime = endTime + + + #Added----------------- + self.selBlocksize = blocksize + self.selBlocktime = blocktime + + + if not(self.setNextFile()): + if (startDate!=None) and (endDate!=None): + print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()) + elif startDate != None: + print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime()) + else: + print "[Reading] No files" + self.fileIndex = -1 self.pathList = [] self.filenameList = [] return # self.getBasicHeader() - + if last_set != None: self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock return @@ -1365,10 +1442,10 @@ class JRODataReader(JRODataIO): self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs - -# self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs - - + + # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs + + def getFirstHeader(self): raise NotImplementedError @@ -1416,30 +1493,29 @@ class JRODataReader(JRODataIO): self.__printInfo = False - def run(self, - path=None, - startDate=None, - endDate=None, - startTime=datetime.time(0,0,0), - endTime=datetime.time(23,59,59), - set=None, - expLabel = "", - ext = None, - online = False, - delay = 60, - walk = True, - getblock = False, - nTxs = 1, - realtime=False, - blocksize=None, - blocktime=None, - queue=None, - skip=None, - cursor=None, - warnings=True, - server=None, - verbose=True, **kwargs): + path=None, + startDate=None, + endDate=None, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + set=None, + expLabel = "", + ext = None, + online = False, + delay = 60, + walk = True, + getblock = False, + nTxs = 1, + realtime=False, + blocksize=None, + blocktime=None, + queue=None, + skip=None, + cursor=None, + warnings=True, + server=None, + verbose=True, **kwargs): if not(self.isConfig): # self.dataOut = dataOut @@ -1464,7 +1540,7 @@ class JRODataReader(JRODataIO): cursor=cursor, warnings=warnings, server=server, - verbose=verbose) + verbose=verbose, **kwargs) self.isConfig = True if server is None: self.getData() @@ -1587,9 +1663,9 @@ class JRODataWriter(JRODataIO): Return: None """ - -# CALCULAR PARAMETROS - + + # CALCULAR PARAMETROS + sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader @@ -1716,7 +1792,7 @@ class JRODataWriter(JRODataIO): return 1 - def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4): + def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, verbose=True): """ Setea el tipo de formato en la cual sera guardada la data y escribe el First Header diff --git a/schainpy/model/io/jroIO_heispectra.py b/schainpy/model/io/jroIO_heispectra.py index 06b1037..d6f37ca 100644 --- a/schainpy/model/io/jroIO_heispectra.py +++ b/schainpy/model/io/jroIO_heispectra.py @@ -119,7 +119,6 @@ class Metadata(object): self.parmConfObjList.append(parmConfObj) class FitsWriter(Operation): - def __init__(self, **kwargs): Operation.__init__(self, **kwargs) self.isConfig = False @@ -276,9 +275,9 @@ class FitsWriter(Operation): self.setNextFile() self.writeNextBlock() - def run(self, dataOut, **kwargs): + def run(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None, **kwargs): if not(self.isConfig): - self.setup(dataOut, **kwargs) + self.setup(dataOut, path, dataBlocksPerFile=dataBlocksPerFile, metadatafile=metadatafile, **kwargs) self.isConfig = True self.putData() diff --git a/schainpy/model/io/jroIO_param.py b/schainpy/model/io/jroIO_param.py index b3fed36..604558f 100644 --- a/schainpy/model/io/jroIO_param.py +++ b/schainpy/model/io/jroIO_param.py @@ -607,28 +607,18 @@ class ParamWriter(Operation): self.isConfig = False return - def setup(self, dataOut, **kwargs): - - self.path = kwargs['path'] - - if kwargs.has_key('blocksPerFile'): - self.blocksPerFile = kwargs['blocksPerFile'] - else: - self.blocksPerFile = 10 - - self.metadataList = kwargs['metadataList'] - self.dataList = kwargs['dataList'] + def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs): + self.path = path + self.blocksPerFile = blocksPerFile + self.metadataList = metadataList + self.dataList = dataList self.dataOut = dataOut - - if kwargs.has_key('mode'): - mode = kwargs['mode'] - - if type(mode) == int: - mode = numpy.zeros(len(self.dataList)) + mode - else: - mode = numpy.ones(len(self.dataList)) - self.mode = mode + + if self.mode is not None: + self.mode = numpy.zeros(len(self.dataList)) + mode + else: + self.mode = numpy.ones(len(self.dataList)) arrayDim = numpy.zeros((len(self.dataList),5)) @@ -770,13 +760,20 @@ class ParamWriter(Operation): else: setFile = -1 #inicializo mi contador de seteo - setFile += 1 - - file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar, - timeTuple.tm_year, - timeTuple.tm_yday, - setFile, - ext ) + if self.setType is None: + setFile += 1 + file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar, + timeTuple.tm_year, + timeTuple.tm_yday, + setFile, + ext ) + else: + setFile = timeTuple.tm_hour*60+timeTuple.tm_min + file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar, + timeTuple.tm_year, + timeTuple.tm_yday, + setFile, + ext ) filename = os.path.join( path, subfolder, file ) self.metaFile = file @@ -849,13 +846,20 @@ class ParamWriter(Operation): os.makedirs(fullpath) setFile = -1 #inicializo mi contador de seteo - setFile += 1 - - file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, - timeTuple.tm_year, - timeTuple.tm_yday, - setFile, - ext ) + if self.setType is None: + setFile += 1 + file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar, + timeTuple.tm_year, + timeTuple.tm_yday, + setFile, + ext ) + else: + setFile = timeTuple.tm_hour*60+timeTuple.tm_min + file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar, + timeTuple.tm_year, + timeTuple.tm_yday, + setFile, + ext ) filename = os.path.join( path, subfolder, file ) @@ -1074,10 +1078,11 @@ class ParamWriter(Operation): self.fp.close() return - def run(self, dataOut, **kwargs): + def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs): if not(self.isConfig): - flagdata = self.setup(dataOut, **kwargs) + flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile, + metadataList=metadataList, dataList=dataList, mode=mode, **kwargs) if not(flagdata): return diff --git a/schainpy/model/io/jroIO_voltage.py b/schainpy/model/io/jroIO_voltage.py index 74e3a9d..d4eeb7f 100644 --- a/schainpy/model/io/jroIO_voltage.py +++ b/schainpy/model/io/jroIO_voltage.py @@ -177,13 +177,13 @@ class VoltageReader(JRODataReader, ProcessingUnit): def getBlockDimension(self): """ - Obtiene la cantidad de puntos a leer por cada bloque de datos - - Affected: - self.blocksize + Obtiene la cantidad de puntos a leer por cada bloque de datos + + Affected: + self.blocksize - Return: - None + Return: + None """ pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels self.blocksize = pts2read @@ -191,27 +191,37 @@ class VoltageReader(JRODataReader, ProcessingUnit): def readBlock(self): """ - readBlock lee el bloque de datos desde la posicion actual del puntero del archivo - (self.fp) y actualiza todos los parametros relacionados al bloque de datos - (metadata + data). La data leida es almacenada en el buffer y el contador del buffer - es seteado a 0 - - Inputs: - None + readBlock lee el bloque de datos desde la posicion actual del puntero del archivo + (self.fp) y actualiza todos los parametros relacionados al bloque de datos + (metadata + data). La data leida es almacenada en el buffer y el contador del buffer + es seteado a 0 - Return: - None - - Affected: - self.profileIndex - self.datablock - self.flagIsNewFile - self.flagIsNewBlock - self.nTotalBlocks + Inputs: + None + + Return: + None - Exceptions: - Si un bloque leido no es un bloque valido + Affected: + self.profileIndex + self.datablock + self.flagIsNewFile + self.flagIsNewBlock + self.nTotalBlocks + + Exceptions: + Si un bloque leido no es un bloque valido """ + + # if self.server is not None: + # self.zBlock = self.receiver.recv() + # self.zHeader = self.zBlock[:24] + # self.zDataBlock = self.zBlock[24:] + # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real',' 1 then one profile is divided by nTxs and number of total - blocks is increased by nTxs (nProfiles *= nTxs) + Return profile by profile + + If nTxs > 1 then one profile is divided by nTxs and number of total + blocks is increased by nTxs (nProfiles *= nTxs) """ self.dataOut.flagDataAsBlock = False self.dataOut.data = self.datablock[:,self.profileIndex,:] @@ -373,19 +474,19 @@ class VoltageReader(JRODataReader, ProcessingUnit): self.profileIndex += 1 -# elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles: -# """ -# Return all block -# """ -# self.dataOut.flagDataAsBlock = True -# self.dataOut.data = self.datablock -# self.dataOut.profileIndex = self.dataOut.nProfiles - 1 -# -# self.profileIndex = self.dataOut.nProfiles + # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles: + # """ + # Return all block + # """ + # self.dataOut.flagDataAsBlock = True + # self.dataOut.data = self.datablock + # self.dataOut.profileIndex = self.dataOut.nProfiles - 1 + # + # self.profileIndex = self.dataOut.nProfiles else: """ - Return a block + Return a block """ if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles if self.selBlocktime != None: diff --git a/schainpy/model/proc/extensions.c b/schainpy/model/proc/extensions.c index 27f0c9d..8f74614 100644 --- a/schainpy/model/proc/extensions.c +++ b/schainpy/model/proc/extensions.c @@ -1,12 +1,22 @@ +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#define NUM_CPY_THREADS 8 #include #include #include +#include +#include +// void printArr(int *array); static PyObject *hildebrand_sekhon(PyObject *self, PyObject *args); +static PyObject *correlateByBlock(PyObject *self, PyObject *args); +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif static PyMethodDef extensionsMethods[] = { - { "hildebrand_sekhon", (PyCFunction)hildebrand_sekhon, METH_VARARGS, "get noise with" }, - { NULL, NULL, 0, NULL } + { "correlateByBlock", (PyCFunction)correlateByBlock, METH_VARARGS, "get correlation by block" }, + { "hildebrand_sekhon", (PyCFunction)hildebrand_sekhon, METH_VARARGS, "get noise with hildebrand_sekhon" }, + { NULL, NULL, 0, NULL } }; PyMODINIT_FUNC initcSchain() { @@ -14,13 +24,83 @@ PyMODINIT_FUNC initcSchain() { import_array(); } +static PyObject *correlateByBlock(PyObject *self, PyObject *args) { + + // int *x = (int*) malloc(4000000 * 216 * sizeof(int));; + // int a = 5; + // x = &a; + // int b = 6; + // x = &b; + // printf("Antes de imprimir x \n"); + // printf("%d \n", x[0]); + + PyObject *data_obj1, *data_obj2; + PyArrayObject *data_array1, *data_array2, *correlateRow, *out, *dataRow, *codeRow; //, , + int mode; + + if (!PyArg_ParseTuple(args, "OOi", &data_obj1, &data_obj2, &mode)) return NULL; + + data_array1 = (PyArrayObject *) PyArray_FROM_OTF(data_obj1, NPY_COMPLEX128, NPY_ARRAY_DEFAULT); + data_array2 = (PyArrayObject *) PyArray_FROM_OTF(data_obj2, NPY_FLOAT64, NPY_ARRAY_DEFAULT); + + npy_intp dims[1]; + dims[0] = 200; + npy_intp dims_code[1]; + dims_code[0] = 16; + + double complex * dataRaw; + double * codeRaw; + dataRaw = (double complex*) PyArray_DATA(data_array1); + codeRaw = (double *) PyArray_DATA(data_array2); + double complex ** outC = malloc(40000*200*sizeof(double complex)); + int i; + + clock_t start = clock(); + for(i=0; i<40000; i++){ + // codeRow = PyArray_SimpleNewFromData(1, dims_code, NPY_FLOAT64, codeRaw + 16 * i); + // dataRow = PyArray_SimpleNewFromData(1, dims, NPY_COMPLEX128, dataRaw + 200 * i); + // Py_INCREF(codeRow); + // Py_INCREF(dataRow); + // PyArray_ENABLEFLAGS(codeRow, NPY_ARRAY_OWNDATA); + // PyArray_ENABLEFLAGS(dataRow, NPY_ARRAY_OWNDATA); + correlateRow = (PyArrayObject *) PyArray_Correlate2(PyArray_SimpleNewFromData(1, dims_code, NPY_FLOAT64, codeRaw + 16 * i), PyArray_SimpleNewFromData(1, dims, NPY_COMPLEX128, dataRaw + 200 * i), (npy_intp) 2); + //Py_INCREF(correlateRow); + // PyArray_ENABLEFLAGS(correlateRow, NPY_ARRAY_OWNDATA); + memcpy(outC + 200*i, (double complex*) PyArray_DATA(correlateRow), 200 * sizeof(double complex)); + + Py_DECREF(correlateRow); + // Py_DECREF(codeRow); + // Py_DECREF(dataRow); + } + clock_t end = clock(); + float seconds = (float)(end - start) / CLOCKS_PER_SEC; + printf("%f", seconds); + // + npy_intp dimsret[2]; + dimsret[0] = 40000; + dimsret[1] = 200; + out = PyArray_SimpleNewFromData(2, dimsret, NPY_COMPLEX128, outC); + PyArray_ENABLEFLAGS(out, NPY_ARRAY_OWNDATA); + //Py_INCREF(out); + Py_DECREF(data_array1); + Py_DECREF(data_array2); + // PyArray_DebugPrint(out); + // Py_DECREF(data_obj2); + // Py_DECREF(data_obj1); + // Py_DECREF(codeRow); + // Py_DECREF(dataRow); + // free(dataRaw); + // free(codeRaw); + + return PyArray_Return(out); +} + static PyObject *hildebrand_sekhon(PyObject *self, PyObject *args) { - /* Do your stuff here. */ double navg; PyObject *data_obj, *data_array; if (!PyArg_ParseTuple(args, "Od", &data_obj, &navg)) return NULL; - data_array = PyArray_FROM_OTF(data_obj, NPY_FLOAT64, NPY_IN_ARRAY); + data_array = PyArray_FROM_OTF(data_obj, NPY_FLOAT64, NPY_ARRAY_DEFAULT); if (data_array == NULL) { Py_XDECREF(data_array); Py_XDECREF(data_obj); @@ -56,3 +136,4 @@ static PyObject *hildebrand_sekhon(PyObject *self, PyObject *args) { return Py_BuildValue("d", lnoise); } + diff --git a/schainpy/model/proc/jroproc_base.py b/schainpy/model/proc/jroproc_base.py index 55f3afd..3b87703 100644 --- a/schainpy/model/proc/jroproc_base.py +++ b/schainpy/model/proc/jroproc_base.py @@ -200,7 +200,6 @@ class ProcessingUnit(object): return True def call(self, opType, opName=None, opId=None): - """ Return True si ejecuta la operacion interna nombrada "opName" o la operacion externa identificada con el id "opId"; con los argumentos "**kwargs". diff --git a/schainpy/model/proc/jroproc_correlation.py b/schainpy/model/proc/jroproc_correlation.py index 716274c..67c72d3 100644 --- a/schainpy/model/proc/jroproc_correlation.py +++ b/schainpy/model/proc/jroproc_correlation.py @@ -174,5 +174,5 @@ class CorrelationProc(ProcessingUnit): self.dataOut.lagRange = numpy.array(lags)*delta # self.dataOut.nCohInt = self.dataIn.nCohInt*nAvg self.dataOut.flagNoData = False - a = self.dataOut.normFactor +# a = self.dataOut.normFactor return diff --git a/schainpy/model/proc/jroproc_heispectra.py b/schainpy/model/proc/jroproc_heispectra.py index e9a25fc..1c6976f 100644 --- a/schainpy/model/proc/jroproc_heispectra.py +++ b/schainpy/model/proc/jroproc_heispectra.py @@ -163,7 +163,6 @@ class IncohInt4SpectraHeis(Operation): n = None - def __init__(self, **kwargs): Operation.__init__(self, **kwargs) @@ -324,10 +323,10 @@ class IncohInt4SpectraHeis(Operation): return avgdata, avgdatatime - def run(self, dataOut, **kwargs): + def run(self, dataOut, n=None, timeInterval=None, overlapping=False, **kwargs): if not self.isConfig: - self.setup(**kwargs) + self.setup(n=n, timeInterval=timeInterval, overlapping=overlapping) self.isConfig = True avgdata, avgdatatime = self.integrate(dataOut.data_spc, dataOut.utctime) diff --git a/schainpy/model/proc/jroproc_parameters.py b/schainpy/model/proc/jroproc_parameters.py index 3111353..18f2ced 100644 --- a/schainpy/model/proc/jroproc_parameters.py +++ b/schainpy/model/proc/jroproc_parameters.py @@ -1017,33 +1017,55 @@ class WindProfiler(Operation): def techniqueNSM_DBS(self, **kwargs): metArray = kwargs['metArray'] heightList = kwargs['heightList'] - timeList = kwargs['timeList'] - zenithList = kwargs['zenithList'] + timeList = kwargs['timeList'] + azimuth = kwargs['azimuth'] + theta_x = numpy.array(kwargs['theta_x']) + theta_y = numpy.array(kwargs['theta_y']) + + utctime = metArray[:,0] + cmet = metArray[:,1].astype(int) + hmet = metArray[:,3].astype(int) + SNRmet = metArray[:,4] + vmet = metArray[:,5] + spcmet = metArray[:,6] + nChan = numpy.max(cmet) + 1 nHeights = len(heightList) - utctime = metArray[:,0] - cmet = metArray[:,1] - hmet = metArray1[:,3].astype(int) - h1met = heightList[hmet]*zenithList[cmet] - vmet = metArray1[:,5] + azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw = self.__calculateAngles(theta_x, theta_y, azimuth) + hmet = heightList[hmet] + h1met = hmet*numpy.cos(zenith_arr[cmet]) #Corrected heights + + velEst = numpy.zeros((heightList.size,2))*numpy.nan for i in range(nHeights - 1): hmin = heightList[i] hmax = heightList[i + 1] - vthisH = vmet[(h1met>=hmin) & (h1met=hmin) & (h1met8) & (vmet<50) & (spcmet<10) + indthisH = numpy.where(thisH) + + if numpy.size(indthisH) > 3: + + vel_aux = vmet[thisH] + chan_aux = cmet[thisH] + cosu_aux = dir_cosu[chan_aux] + cosv_aux = dir_cosv[chan_aux] + cosw_aux = dir_cosw[chan_aux] + + nch = numpy.size(numpy.unique(chan_aux)) + if nch > 1: + A = self.__calculateMatA(cosu_aux, cosv_aux, cosw_aux, True) + velEst[i,:] = numpy.dot(A,vel_aux) + + return velEst - def run(self, dataOut, technique, hmin=70, hmax=110, nHours=1, **kwargs): + def run(self, dataOut, technique, nHours=1, hmin=70, hmax=110, **kwargs): param = dataOut.data_param if dataOut.abscissaList != None: absc = dataOut.abscissaList[:-1] - #noise = dataOut.noise + # noise = dataOut.noise heightList = dataOut.heightList SNR = dataOut.data_SNR @@ -1153,11 +1175,15 @@ class WindProfiler(Operation): else: rx_location = [(0,1),(1,1),(1,0)] if kwargs.has_key('azimuth'): azimuth = kwargs['azimuth'] - else: azimuth = 51 + else: azimuth = 51.06 if kwargs.has_key('dfactor'): dfactor = kwargs['dfactor'] if kwargs.has_key('mode'): mode = kwargs['mode'] + if kwargs.has_key('theta_x'): + theta_x = kwargs['theta_x'] + if kwargs.has_key('theta_y'): + theta_y = kwargs['theta_y'] else: mode = 'SA' #Borrar luego esto @@ -1200,7 +1226,7 @@ class WindProfiler(Operation): if mode == 'SA': dataOut.data_output = self.techniqueNSM_SA(rx_location=rx_location, groupList=groupList, azimuth=azimuth, dfactor=dfactor, k=k,metArray=metArray, heightList=heightList,timeList=timeList) elif mode == 'DBS': - dataOut.data_output = self.techniqueNSM_DBS(metArray=metArray,heightList=heightList,timeList=timeList) + dataOut.data_output = self.techniqueNSM_DBS(metArray=metArray,heightList=heightList,timeList=timeList, azimuth=azimuth, theta_x=theta_x, theta_y=theta_y) dataOut.data_output = dataOut.data_output.T dataOut.flagNoData = False self.__buffer = None @@ -1277,25 +1303,26 @@ class EWDriftsEstimation(Operation): class NonSpecularMeteorDetection(Operation): - def run(self, mode, SNRthresh=8, phaseDerThresh=0.5, cohThresh=0.8, allData = False): - data_acf = self.dataOut.data_pre[0] - data_ccf = self.dataOut.data_pre[1] - - lamb = self.dataOut.C/self.dataOut.frequency - tSamp = self.dataOut.ippSeconds*self.dataOut.nCohInt - paramInterval = self.dataOut.paramInterval - + def run(self, dataOut, mode, SNRthresh=8, phaseDerThresh=0.5, cohThresh=0.8, allData = False): + data_acf = dataOut.data_pre[0] + data_ccf = dataOut.data_pre[1] + pairsList = dataOut.groupList[1] + + lamb = dataOut.C/dataOut.frequency + tSamp = dataOut.ippSeconds*dataOut.nCohInt + paramInterval = dataOut.paramInterval + nChannels = data_acf.shape[0] nLags = data_acf.shape[1] nProfiles = data_acf.shape[2] - nHeights = self.dataOut.nHeights - nCohInt = self.dataOut.nCohInt - sec = numpy.round(nProfiles/self.dataOut.paramInterval) - heightList = self.dataOut.heightList - ippSeconds = self.dataOut.ippSeconds*self.dataOut.nCohInt*self.dataOut.nAvg - utctime = self.dataOut.utctime - - self.dataOut.abscissaList = numpy.arange(0,paramInterval+ippSeconds,ippSeconds) + nHeights = dataOut.nHeights + nCohInt = dataOut.nCohInt + sec = numpy.round(nProfiles/dataOut.paramInterval) + heightList = dataOut.heightList + ippSeconds = dataOut.ippSeconds*dataOut.nCohInt*dataOut.nAvg + utctime = dataOut.utctime + + dataOut.abscissaList = numpy.arange(0,paramInterval+ippSeconds,ippSeconds) #------------------------ SNR -------------------------------------- power = data_acf[:,0,:,:].real @@ -1308,6 +1335,7 @@ class NonSpecularMeteorDetection(Operation): SNRdB = 10*numpy.log10(SNR) if mode == 'SA': + dataOut.groupList = dataOut.groupList[1] nPairs = data_ccf.shape[0] #---------------------- Coherence and Phase -------------------------- phase = numpy.zeros(data_ccf[:,0,:,:].shape) @@ -1315,8 +1343,8 @@ class NonSpecularMeteorDetection(Operation): coh1 = numpy.zeros(data_ccf[:,0,:,:].shape) for p in range(nPairs): - ch0 = self.dataOut.groupList[p][0] - ch1 = self.dataOut.groupList[p][1] + ch0 = pairsList[p][0] + ch1 = pairsList[p][1] ccf = data_ccf[p,0,:,:]/numpy.sqrt(data_acf[ch0,0,:,:]*data_acf[ch1,0,:,:]) phase[p,:,:] = ndimage.median_filter(numpy.angle(ccf), size = (5,1)) #median filter # phase1[p,:,:] = numpy.angle(ccf) #median filter @@ -1388,16 +1416,18 @@ class NonSpecularMeteorDetection(Operation): data_param[:,6:] = phase[:,tmet,hmet].T elif mode == 'DBS': - self.dataOut.groupList = numpy.arange(nChannels) + dataOut.groupList = numpy.arange(nChannels) #Radial Velocities -# phase = numpy.angle(data_acf[:,1,:,:]) - phase = ndimage.median_filter(numpy.angle(data_acf[:,1,:,:]), size = (1,5,1)) + phase = numpy.angle(data_acf[:,1,:,:]) +# phase = ndimage.median_filter(numpy.angle(data_acf[:,1,:,:]), size = (1,5,1)) velRad = phase*lamb/(4*numpy.pi*tSamp) #Spectral width - acf1 = ndimage.median_filter(numpy.abs(data_acf[:,1,:,:]), size = (1,5,1)) - acf2 = ndimage.median_filter(numpy.abs(data_acf[:,2,:,:]), size = (1,5,1)) +# acf1 = ndimage.median_filter(numpy.abs(data_acf[:,1,:,:]), size = (1,5,1)) +# acf2 = ndimage.median_filter(numpy.abs(data_acf[:,2,:,:]), size = (1,5,1)) + acf1 = data_acf[:,1,:,:] + acf2 = data_acf[:,2,:,:] spcWidth = (lamb/(2*numpy.sqrt(6)*numpy.pi*tSamp))*numpy.sqrt(numpy.log(acf1/acf2)) # velRad = ndimage.median_filter(velRad, size = (1,5,1)) @@ -1409,7 +1439,7 @@ class NonSpecularMeteorDetection(Operation): boolMet1 = ndimage.median_filter(boolMet1, size=(1,5,5)) #Radial velocity - boolMet2 = numpy.abs(velRad) < 30 + boolMet2 = numpy.abs(velRad) < 20 boolMet2 = ndimage.median_filter(boolMet2, (1,5,5)) #Spectral Width @@ -1436,9 +1466,9 @@ class NonSpecularMeteorDetection(Operation): # self.dataOut.data_param = data_int if len(data_param) == 0: - self.dataOut.flagNoData = True + dataOut.flagNoData = True else: - self.dataOut.data_param = data_param + dataOut.data_param = data_param def __erase_small(self, binArray, threshX, threshY): labarray, numfeat = ndimage.measurements.label(binArray) @@ -2245,10 +2275,10 @@ class SMPhaseCalibration(Operation): pairi = pairs[i] - phip3 = phases[:,pairi[1]] - d3 = d[pairi[1]] - phip2 = phases[:,pairi[0]] - d2 = d[pairi[0]] + phip3 = phases[:,pairi[0]] + d3 = d[pairi[0]] + phip2 = phases[:,pairi[1]] + d2 = d[pairi[1]] #Calculating gamma # jdcos = alp1/(k*d1) # jgamma = numpy.angle(numpy.exp(1j*(d0*alp1/d1 - alp0))) @@ -2261,7 +2291,7 @@ class SMPhaseCalibration(Operation): jgammaArray = numpy.hstack((jgamma,jgamma+0.5*numpy.pi,jgamma-0.5*numpy.pi)) #Histogram - nBins = 64.0 + nBins = 64 rmin = -0.5*numpy.pi rmax = 0.5*numpy.pi phaseHisto = numpy.histogram(jgammaArray, bins=nBins, range=(rmin,rmax)) @@ -2303,15 +2333,15 @@ class SMPhaseCalibration(Operation): def __getPhases(self, azimuth, h, pairsList, d, gammas, meteorsArray): meteorOps = SMOperations() nchan = 4 - pairx = pairsList[0] - pairy = pairsList[1] + pairx = pairsList[0] #x es 0 + pairy = pairsList[1] #y es 1 center_xangle = 0 center_yangle = 0 range_angle = numpy.array([10*numpy.pi,numpy.pi,numpy.pi/2,numpy.pi/4]) ntimes = len(range_angle) - nstepsx = 20.0 - nstepsy = 20.0 + nstepsx = 20 + nstepsy = 20 for iz in range(ntimes): min_xangle = -range_angle[iz]/2 + center_xangle @@ -2331,14 +2361,28 @@ class SMPhaseCalibration(Operation): # Iterations looking for the offset for iy in range(int(nstepsy)): for ix in range(int(nstepsx)): - jph[pairy[1]] = alpha_y[iy] - jph[pairy[0]] = -gammas[1] - alpha_y[iy]*d[pairy[1]]/d[pairy[0]] - - jph[pairx[1]] = alpha_x[ix] - jph[pairx[0]] = -gammas[0] - alpha_x[ix]*d[pairx[1]]/d[pairx[0]] - + d3 = d[pairsList[1][0]] + d2 = d[pairsList[1][1]] + d5 = d[pairsList[0][0]] + d4 = d[pairsList[0][1]] + + alp2 = alpha_y[iy] #gamma 1 + alp4 = alpha_x[ix] #gamma 0 + + alp3 = -alp2*d3/d2 - gammas[1] + alp5 = -alp4*d5/d4 - gammas[0] +# jph[pairy[1]] = alpha_y[iy] +# jph[pairy[0]] = -gammas[1] - alpha_y[iy]*d[pairy[1]]/d[pairy[0]] + +# jph[pairx[1]] = alpha_x[ix] +# jph[pairx[0]] = -gammas[0] - alpha_x[ix]*d[pairx[1]]/d[pairx[0]] + jph[pairsList[0][1]] = alp4 + jph[pairsList[0][0]] = alp5 + jph[pairsList[1][0]] = alp3 + jph[pairsList[1][1]] = alp2 jph_array[:,ix,iy] = jph - +# d = [2.0,2.5,2.5,2.0] + #falta chequear si va a leer bien los meteoros meteorsArray1 = meteorOps.getMeteorParams(meteorsArray, azimuth, h, pairsList, d, jph) error = meteorsArray1[:,-1] ind1 = numpy.where(error==0)[0] @@ -2387,7 +2431,8 @@ class SMPhaseCalibration(Operation): k = 2*numpy.pi/lamb azimuth = 0 h = (hmin, hmax) - pairs = ((0,1),(2,3)) +# pairs = ((0,1),(2,3)) #Estrella +# pairs = ((1,0),(2,3)) #T if channelPositions is None: # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T @@ -2395,6 +2440,17 @@ class SMPhaseCalibration(Operation): meteorOps = SMOperations() pairslist0, distances = meteorOps.getPhasePairs(channelPositions) + #Checking correct order of pairs + pairs = [] + if distances[1] > distances[0]: + pairs.append((1,0)) + else: + pairs.append((0,1)) + + if distances[3] > distances[2]: + pairs.append((3,2)) + else: + pairs.append((2,3)) # distances1 = [-distances[0]*lamb, distances[1]*lamb, -distances[2]*lamb, distances[3]*lamb] meteorsArray = self.__buffer @@ -2545,8 +2601,8 @@ class SMOperations(): hCorr = hi[ind_h, :] ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight)) - - hCorr = hi[ind_hCorr] + + hCorr = hi[ind_hCorr][:len(ind_h)] heights[ind_h] = hCorr #Setting Error diff --git a/schainpy/model/proc/jroproc_voltage.py b/schainpy/model/proc/jroproc_voltage.py index 9f84851..84c4ace 100644 --- a/schainpy/model/proc/jroproc_voltage.py +++ b/schainpy/model/proc/jroproc_voltage.py @@ -1,9 +1,11 @@ import sys import numpy +from profilehooks import profile from scipy import interpolate - +from schainpy import cSchain from jroproc_base import ProcessingUnit, Operation from schainpy.model.data.jrodata import Voltage +from time import time class VoltageProc(ProcessingUnit): @@ -332,7 +334,6 @@ class CohInt(Operation): n = None - def __init__(self, **kwargs): Operation.__init__(self, **kwargs) @@ -345,10 +346,9 @@ class CohInt(Operation): Inputs: - n : Number of coherent integrations - timeInterval : Time of integration. If the parameter "n" is selected this one does not work - overlapping : - + n : Number of coherent integrations + timeInterval : Time of integration. If the parameter "n" is selected this one does not work + overlapping : """ self.__initime = None @@ -548,14 +548,13 @@ class Decoder(Operation): nCode = None nBaud = None - def __init__(self, **kwargs): Operation.__init__(self, **kwargs) self.times = None self.osamp = None -# self.__setValues = False + # self.__setValues = False self.isConfig = False def setup(self, code, osamp, dataOut): @@ -624,19 +623,41 @@ class Decoder(Operation): return self.datadecTime + #@profile + def oldCorrelate(self, i, data, code_block): + profilesList = xrange(self.__nProfiles) + for j in profilesList: + self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:] + + #@profile def __convolutionByBlockInTime(self, data): repetitions = self.__nProfiles / self.nCode - + junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize)) junk = junk.flatten() code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud)) - - for i in range(self.__nChannels): - for j in range(self.__nProfiles): + profilesList = xrange(self.__nProfiles) + + # def toVectorize(a,b): + # return numpy.correlate(a,b, mode='full') + # vectorized = numpy.vectorize(toVectorize, signature='(n),(m)->(k)') + for i in range(self.__nChannels): + # self.datadecTime[i,:,:] = numpy.array([numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:] for j in profilesList ]) + # def func(i, j): + # self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:] + # map(lambda j: func(i, j), range(self.__nProfiles)) + #print data[i,:,:].shape + # self.datadecTime[i,:,:] = vectorized(data[i,:,:], code_block[:,:])[:,self.nBaud-1:] + for j in profilesList: self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:] - + # print data[i,:,:] + # print cSchain.correlateByBlock(data[i,:,:], code_block, 2) + # self.datadecTime[i,:,:] = cSchain.correlateByBlock(data[i,:,:], code_block, 2) + # print self.datadecTime[i,:,:] + #print self.datadecTime[i,:,:].shape return self.datadecTime + def __convolutionByBlockInFreq(self, data): @@ -653,6 +674,7 @@ class Decoder(Operation): return data + def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None): if dataOut.flagDecodeData: @@ -682,7 +704,9 @@ class Decoder(Operation): print "Fail decoding: Code is not defined." return + self.__nProfiles = dataOut.nProfiles datadec = None + if mode == 3: mode = 0 @@ -1085,7 +1109,6 @@ class SplitProfiles(Operation): dataOut.ippSeconds /= n class CombineProfiles(Operation): - def __init__(self, **kwargs): Operation.__init__(self, **kwargs) diff --git a/schainpy/model/utils/jroutils_ftp.py b/schainpy/model/utils/jroutils_ftp.py index d2a1264..a507f2f 100644 --- a/schainpy/model/utils/jroutils_ftp.py +++ b/schainpy/model/utils/jroutils_ftp.py @@ -196,7 +196,7 @@ class Remote(Thread): if self.stopFlag: break - # self.bussy = True + # self.bussy = True self.mutex.acquire() print "[Remote Server] Opening %s" %self.__server @@ -211,7 +211,7 @@ class Remote(Thread): self.close() self.mutex.release() -# self.bussy = False + # self.bussy = False print "[Remote Server] Thread stopped successfully" @@ -619,6 +619,7 @@ class SendToServer(ProcessingUnit): filenameList = glob.glob1(thisFolder, '*%s' %self.ext) if len(filenameList) < 1: + continue for thisFile in filenameList: @@ -706,7 +707,7 @@ class FTP(object): self.ftp = ftplib.FTP(self.server) self.ftp.login(self.username,self.password) self.ftp.cwd(self.remotefolder) - # print 'Connect to FTP Server: Successfully' + # print 'Connect to FTP Server: Successfully' except ftplib.all_errors: print 'Error FTP Service' diff --git a/schainpy/scripts/JASMET30_MetDet.py b/schainpy/scripts/JASMET30_MetDet.py index 2286c39..106a7a8 100644 --- a/schainpy/scripts/JASMET30_MetDet.py +++ b/schainpy/scripts/JASMET30_MetDet.py @@ -1,11 +1,6 @@ import os, sys -path = os.path.split(os.getcwd())[0] -path = os.path.split(path)[0] - -sys.path.insert(0, path) - from schainpy.controller import Project controllerObj = Project() @@ -18,7 +13,7 @@ controllerObj.setup(id = '002', name='script02', description="JASMET Meteor Dete # path = '/mnt/jars/2016_08/NOCHE' # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/DIA' # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/NOCHE' -path = '/media/nanosat/NewVolumen/JASMET/2016_08/DIA' +path = '/home/nanosat/data/jasmet' #Path para los graficos pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/graphics') @@ -27,8 +22,8 @@ pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/graphics') pathfile = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/meteor') #Fechas para busqueda de archivos -startDate = '2016/08/29' -endDate = '2016/09/11' +startDate = '2010/08/29' +endDate = '2017/09/11' #Horas para busqueda de archivos startTime = '00:00:00' endTime = '23:59:59' @@ -60,15 +55,23 @@ opObj00.addParameter(name='channelList', value='0,1,2,3,4', format='intlist') opObj01 = procUnitConfObj0.addOperation(name='setRadarFrequency') opObj01.addParameter(name='frequency', value='30.e6', format='float') -opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights') -opObj01.addParameter(name='topLim', value='73', format='int') -opObj01.addParameter(name='botLim', value='71', format='int') +# opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights') +# opObj01.addParameter(name='topLim', value='73', format='int') +# opObj01.addParameter(name='botLim', value='71', format='int') opObj02 = procUnitConfObj0.addOperation(name='Decoder', optype='other') opObj03 = procUnitConfObj0.addOperation(name='CohInt', optype='other') opObj03.addParameter(name='n', value='2', format='int') +procUnitConfObj1 = controllerObj.addProcUnit(datatype='SpectraProc', inputId=procUnitConfObj0.getId()) +opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other') +opObj11.addParameter(name='id', value='237', format='int') +opObj11.addParameter(name='xmin', value='9.0', format='float') +opObj11.addParameter(name='xmax', value='16.0', format='float') +opObj11.addParameter(name='zmin', value='15.0', format='float') +opObj11.addParameter(name='zmax', value='50.0', format='float') + #--------------------------- Parameters Processing Unit ------------------------------------ procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId()) @@ -84,14 +87,9 @@ opObj12.addParameter(name='blocksPerFile', value='1000', format='int') opObj12.addParameter(name='metadataList',value='type,heightList,paramInterval,timeZone',format='list') opObj12.addParameter(name='dataList',value='data_param,utctime',format='list') opObj12.addParameter(name='mode',value='2',format='int') - + #-------------------------------------------------------------------------------------------------- -print "Escribiendo el archivo XML" -controllerObj.writeXml("JASMET02.xml") -print "Leyendo el archivo XML" -controllerObj.readXml("JASMET02.xml") +controllerObj.start() + -controllerObj.createObjects() -controllerObj.connectObjects() -controllerObj.run() \ No newline at end of file diff --git a/schainpy/scripts/JASMET30_PhaseCal.py b/schainpy/scripts/JASMET30_PhaseCal.py new file mode 100644 index 0000000..1b0da1f --- /dev/null +++ b/schainpy/scripts/JASMET30_PhaseCal.py @@ -0,0 +1,68 @@ +import os, sys + +path = os.path.split(os.getcwd())[0] +path = os.path.split(path)[0] + +sys.path.insert(0, path) + +from schainpy.controller import Project + +controllerObj = Project() +controllerObj.setup(id = '004', name='script04', description="JASMET Phase Calibration") + +#-------------------------------------- Setup ----------------------------------------- +#Verificar estas variables + +#Path donde estan los archivos HDF5 de meteoros +path = os.path.join(os.environ['HOME'],'Pictures/JASMET30_mp/201608/meteor') + +#Path para los graficos +pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30_mp/201608/graphics') + +#Path donde se almacenaran las fases calculadas +pathfile = os.path.join(os.environ['HOME'],'Pictures/JASMET30_mp/201608/phase') + +#Fechas para busqueda de archivos +startDate = '2016/08/29' +endDate = '2016/09/11' +#Horas para busqueda de archivos +startTime = '00:00:00' +endTime = '23:59:59' + +#------------------------------------------------------------------------------------------------ +readUnitConfObj = controllerObj.addReadUnit(datatype='ParamReader', + path=path, + startDate=startDate, + endDate=endDate, + startTime=startTime, + endTime=endTime, + walk=1) + +#-------------------------------------------------------------------------------------------------- + +procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId()) +# # +opObj31 = procUnitConfObj1.addOperation(name='SMPhaseCalibration', optype='other') +opObj31.addParameter(name='nHours', value='1', format='float') +opObj31.addParameter(name='hmin', value='60', format='float') +opObj31.addParameter(name='hmax', value='120', format='float') +# opObj31.addParameter(name='channelPositions', value='(2.5,0),(0,2.5),(0,0),(0,4.5),(-2,0)', format='pairslist') + +opObj32 = procUnitConfObj1.addOperation(name='PhasePlot', optype='other') +opObj32.addParameter(name='id', value='201', format='int') +opObj32.addParameter(name='wintitle', value='PhaseCalibration', format='str') +opObj32.addParameter(name='save', value='1', format='bool') +opObj32.addParameter(name='xmin', value='0', format='float') +opObj32.addParameter(name='xmax', value='24', format='float') +opObj32.addParameter(name='ymin', value='-180', format='float') +opObj32.addParameter(name='ymax', value='180', format='float') +opObj32.addParameter(name='figpath', value=pathfig, format='str') +# # +opObj33 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other') +opObj33.addParameter(name='path', value=pathfile) +opObj33.addParameter(name='blocksPerFile', value='1000', format='int') +opObj33.addParameter(name='metadataList',value='type,outputInterval,timeZone',format='list') +opObj33.addParameter(name='dataList',value='data_output,utctime',format='list') +# # opObj25.addParameter(name='mode',value='1,0,0',format='intlist') + +controllerObj.start() \ No newline at end of file diff --git a/schainpy/scripts/JASMET30_Winds.py b/schainpy/scripts/JASMET30_Winds.py new file mode 100644 index 0000000..f489be5 --- /dev/null +++ b/schainpy/scripts/JASMET30_Winds.py @@ -0,0 +1,85 @@ +import os, sys + +path = os.path.split(os.getcwd())[0] +path = os.path.split(path)[0] + +sys.path.insert(0, path) + +from schainpy.controller import Project + +controllerObj = Project() +controllerObj.setup(id = '005', name='script05', description="JASMET Wind Estimation") + +#-------------------------------------- Setup ----------------------------------------- +#Verificar estas variables + +#Path donde estan los archivos HDF5 de meteoros +path = os.path.join(os.environ['HOME'],'Pictures/JASMET30_mp/201608/meteor') + +#Path para los graficos +pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30_mp/201608/graphics') + +#Path donde se almacenaran las estimaciones de vientos +pathfile = os.path.join(os.environ['HOME'],'Pictures/JASMET30_mp/201608/phase') + +#Fechas para busqueda de archivos +startDate = '2016/08/29' +endDate = '2016/09/11' +#Horas para busqueda de archivos +startTime = '00:00:00' +endTime = '23:59:59' + +#Offsets optimos obtenidos con OptimumOffset.py +phaseOffsets = '-2.84, -1.77, 11.94, 9.71' +phaseOffsets = '-5.86, -0.93, -7.29, 23.35' +#------------------------------------------------------------------------------------------------ +readUnitConfObj = controllerObj.addReadUnit(datatype='ParamReader', + path=path, + startDate=startDate, + endDate=endDate, + startTime=startTime, + endTime=endTime, + walk=1) +#-------------------------------------------------------------------------------------------------- + +procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId()) +opObj10 = procUnitConfObj1.addOperation(name='CorrectSMPhases',optype='other') +opObj10.addParameter(name='phaseOffsets', value=phaseOffsets, format='floatlist') + +opObj13 = procUnitConfObj1.addOperation(name='SkyMapPlot', optype='other') +opObj13.addParameter(name='id', value='1', format='int') +opObj13.addParameter(name='wintitle', value='Sky Map', format='str') +opObj13.addParameter(name='save', value='1', format='bool') +opObj13.addParameter(name='figpath', value=pathfig, format='str') +opObj13.addParameter(name='ftp', value='1', format='int') +opObj13.addParameter(name='exp_code', value='15', format='int') +opObj13.addParameter(name='sub_exp_code', value='1', format='int') +opObj13.addParameter(name='tmin', value='0', format='int') +opObj13.addParameter(name='tmax', value='24', format='int') + +opObj22 = procUnitConfObj1.addOperation(name='WindProfiler', optype='other') +opObj22.addParameter(name='technique', value='Meteors', format='str') +opObj22.addParameter(name='nHours', value='1', format='float') +opObj22.addParameter(name='hmin', value='70', format='float') +opObj22.addParameter(name='hmax', value='120', format='float') + +opObj23 = procUnitConfObj1.addOperation(name='WindProfilerPlot', optype='other') +opObj23.addParameter(name='id', value='2', format='int') +opObj23.addParameter(name='wintitle', value='Wind Profiler', format='str') +opObj23.addParameter(name='save', value='1', format='bool') +opObj23.addParameter(name='figpath', value = pathfig, format='str') +opObj23.addParameter(name='zmin', value='-140', format='int') +opObj23.addParameter(name='zmax', value='140', format='int') +opObj23.addParameter(name='xmin', value='0', format='float') +opObj23.addParameter(name='xmax', value='24', format='float') +opObj23.addParameter(name='ymin', value='70', format='float') +opObj23.addParameter(name='ymax', value='110', format='float') + +opObj33 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other') +opObj33.addParameter(name='path', value=pathfile) +opObj33.addParameter(name='blocksPerFile', value='1000', format='int') +opObj33.addParameter(name='metadataList',value='type,outputInterval,timeZone',format='list') +opObj33.addParameter(name='dataList',value='data_output,utctime',format='list') +#-------------------------------------------------------------------------------------------------- + +controllerObj.start() \ No newline at end of file diff --git a/schainpy/scripts/PPD.py b/schainpy/scripts/PPD.py index 0ab780a..0c66244 100644 --- a/schainpy/scripts/PPD.py +++ b/schainpy/scripts/PPD.py @@ -11,20 +11,20 @@ def fiber(cursor, skip, q, dt): controllerObj.setup(id='191', name='test01', description=desc) readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader', - path='/home/nanosat/data/sp1_f0', - startDate=dt, - endDate=dt, - startTime="00:00:00", - endTime="23:59:59", - online=0, - #set=1426485881, - walk=1, - queue=q, - cursor=cursor, - skip=skip, - verbose=1 - #timezone=-5*3600 - ) + path='/home/nanosat/data/sp1_f0', + startDate=dt, + endDate=dt, + startTime="00:00:00", + endTime="23:59:59", + online=0, + #set=1426485881, + walk=1, + queue=q, + cursor=cursor, + skip=skip, + verbose=1 + #timezone=-5*3600 + ) # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock') # diff --git a/schainpy/scripts/PPD2.py b/schainpy/scripts/PPD2.py deleted file mode 100644 index f67252c..0000000 --- a/schainpy/scripts/PPD2.py +++ /dev/null @@ -1,88 +0,0 @@ -import argparse - -from schainpy.controller import Project, multiSchain - -desc = "HF_EXAMPLE" - -controllerObj = Project() - -controllerObj.setup(id='191', name='test01', description=desc) - -readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader', - path='/home/nanosat/data/sp1_f0', - startDate="2017/01/26", - endDate="2017/01/26", - startTime="00:00:00", - endTime="23:59:59", - online=0, - #set=1426485881, - walk=1, - verbose=1 - #timezone=-5*3600 - ) - -# #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock') -# -# procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId()) -# procUnitConfObj2.addParameter(name='nipp', value='5', format='int') - -procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId()) -opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other') - -# -# opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other') -# opObj11.addParameter(name='id', value='1000', format='int') -# opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str') -# opObj11.addParameter(name='channelList', value='0', format='intlist') -# opObj11.addParameter(name='zmin', value='-120', format='float') -# opObj11.addParameter(name='zmax', value='-70', format='float') -# opObj11.addParameter(name='save', value='1', format='int') -# opObj11.addParameter(name='figpath', value=figpath, format='str') - -opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other') -# opObj11.addParameter(name='channelList', value='0', format='intList') - -opObj11.addParameter(name='id', value='2000', format='int') -# # opObj11.addParameter(name='colormap', value='0', format='bool') -opObj11.addParameter(name='onlySNR', value='1', format='bool') -opObj11.addParameter(name='DOP', value='0', format='bool') -opObj11.addParameter(name='SNR', value='1', format='bool') -opObj11.addParameter(name='SNRthresh', value='0', format='int') -opObj11.addParameter(name='SNRmin', value='-10', format='int') -opObj11.addParameter(name='SNRmax', value='30', format='int') -opObj11.addParameter(name='xmin', value='0', format='int') -opObj11.addParameter(name='xmax', value='24', format='int') - -# opObj12 = procUnitConfObj3.addOperation(name='ParametersPlot', optype='other') -# #opObj11.addParameter(name='channelList', value='0', format='intlist') -# opObj12.addParameter(name='id', value='301', format='int') -# opObj12.addParameter(name='xmin', value='0', format='float') -# opObj12.addParameter(name='xmax', value='24', format='float') - -# opObj11.addParameter(name='zmin', value='-110', format='float') -# opObj11.addParameter(name='zmax', value='-70', format='float') -# opObj11.addParameter(name='save', value='0', format='int') -# # opObj11.addParameter(name='figpath', value='/tmp/', format='str') -# -# opObj12 = procUnitConfObj3.addOperation(name='PublishData', optype='other') -# opObj12.addParameter(name='zeromq', value=1, format='int') -# opObj12.addParameter(name='verbose', value=0, format='bool') - - -# opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other') -# opObj13.addParameter(name='zeromq', value=1, format='int') -# opObj13.addParameter(name='server', value="juanca", format='str') - -# opObj12.addParameter(name='delay', value=0, format='int') - - -# print "Escribiendo el archivo XML" -# controllerObj.writeXml(filename) -# print "Leyendo el archivo XML" -# controllerObj.readXml(filename) - - -# timeit.timeit('controllerObj.run()', number=2) - -controllerObj.start() - diff --git a/schainpy/scripts/julia_mp.py b/schainpy/scripts/julia_mp.py index 932bd25..2a8eaf1 100644 --- a/schainpy/scripts/julia_mp.py +++ b/schainpy/scripts/julia_mp.py @@ -11,20 +11,20 @@ def fiber(cursor, skip, q, dt): controllerObj.setup(id='191', name='test01', description=desc) readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader', - path='/home/nanosat/data/julia', - startDate=dt, - endDate=dt, - startTime="00:00:00", - endTime="23:59:59", - online=0, - #set=1426485881, - delay=10, - walk=1, - queue=q, - cursor=cursor, - skip=skip, - #timezone=-5*3600 - ) + path='/home/nanosat/data/julia', + startDate=dt, + endDate=dt, + startTime="00:00:00", + endTime="23:59:59", + online=0, + #set=1426485881, + delay=10, + walk=1, + queue=q, + cursor=cursor, + skip=skip, + #timezone=-5*3600 + ) # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock') # diff --git a/schainpy/scripts/optimun_offset.py b/schainpy/scripts/optimun_offset.py new file mode 100644 index 0000000..fa6e660 --- /dev/null +++ b/schainpy/scripts/optimun_offset.py @@ -0,0 +1,117 @@ +import h5py +import numpy +import matplotlib.pyplot as plt +import glob +import os + +#---------------------- Functions --------------------- + +def findFiles(path): + + dirList = [] + fileList = [] + + for thisPath in os.listdir(path): + dirList.append(os.path.join(path,thisPath)) + dirList.sort() + + for thisDirectory in dirList: + files = glob.glob1(thisDirectory, "*.hdf5") + files.sort() + for thisFile in files: + fileList.append(os.path.join(thisDirectory,thisFile)) + + return fileList + +def readFiles(fileList): + + meteors_array = numpy.zeros((1,4)) + + for thisFile in fileList: + + #Leer + f1 = h5py.File(thisFile,'r') + grp1 = f1['Data'] + grp2 = grp1['data_output'] + meteors1 = grp2['table0'][:] + meteors_array = numpy.vstack((meteors_array,meteors1)) + #cerrar + f1.close() + + meteors_array = numpy.delete(meteors_array, 0, axis=0) + meteors_list = [meteors_array[:,0],meteors_array[:,1],meteors_array[:,2],meteors_array[:,3]] + return meteors_list + +def estimateMean(offset_list): + + mean_off = [] + axisY_off = [] + axisX_off = [] + + for thisOffset in offset_list: + mean_aux = numpy.mean(thisOffset, axis = 0) + mean_off.append(mean_aux) + axisX_off.append(numpy.array([0,numpy.size(thisOffset)])) + axisY_off.append(numpy.array([mean_aux,mean_aux])) + + return mean_off, axisY_off, axisX_off + +def plotPhases(offset0, axisY0, axisX0, title): + f, axarr = plt.subplots(4, sharey=True) + color = ['b','g','r','c'] +# plt.grid() + for i in range(len(offset0)): + thisMeteor = offset0[i] + thisY = axisY0[i] + thisX = axisX0[i] + thisColor = color[i] + + opt = thisColor + 'o' + axarr[i].plot(thisMeteor,opt) + axarr[i].plot(thisX, thisY, thisColor) + axarr[i].set_ylabel('Offset ' + str(i)) + + plt.ylim((-180,180)) + axarr[0].set_title(title + ' Offsets') + axarr[3].set_xlabel('Number of estimations') + + return + +def filterOffsets(offsets0, stdvLimit): + offsets1 = [] + + for thisOffset in offsets0: + pstd = numpy.std(thisOffset)*stdvLimit + pmean = numpy.mean(thisOffset) + outlier1 = thisOffset > pmean - pstd + outlier2 = thisOffset < pmean + pstd + not_outlier = numpy.logical_and(outlier1,outlier2) + thisOffset1 = thisOffset[not_outlier] + offsets1.append(thisOffset1) + + return offsets1 + +#---------------------- Setup --------------------------- + +<<<<<<< HEAD +path = '/home/nanosat/Pictures/JASMET30_mp/201608/phase' +======= +path = '/home/jespinoza/Pictures/JASMET30/201608/phase' +>>>>>>> master +stdvLimit = 0.5 + +#---------------------- Script --------------------------- + +fileList = findFiles(path) +offsets0 = readFiles(fileList) +mean0, axisY0, axisX0 = estimateMean(offsets0) +plotPhases(offsets0, axisY0, axisX0, 'Original') + +offsets1 = filterOffsets(offsets0, stdvLimit) +mean1, axisY1, axisX1 = estimateMean(offsets1) +plotPhases(offsets1, axisY1, axisX1, 'Filtered') + +print "Original Offsets: %.2f, %.2f, %.2f, %.2f" % (mean0[0],mean0[1],mean0[2],mean0[3]) +print "Filtered Offsets: %.2f, %.2f, %.2f, %.2f" % (mean1[0],mean1[1],mean1[2],mean1[3]) + +plt.show() diff --git a/schainpy/scripts/project.py b/schainpy/scripts/project.py deleted file mode 100644 index 17237bd..0000000 --- a/schainpy/scripts/project.py +++ /dev/null @@ -1,33 +0,0 @@ -from schainpy.controller import Project - -desc = "A schain project" - -controller = Project() -controller.setup(id='191', name="project", description=desc) - -readUnitConf = controller.addReadUnit(datatype='VoltageReader', - path="/home/nanosat/schain/schainpy/scripts", - startDate="1970/01/01", - endDate="2017/12/31", - startTime="00:00:00", - endTime="23:59:59", - online=0, - walk=1, - ) - -procUnitConf1 = controller.addProcUnit(datatype='VoltageProc', inputId=readUnitConf.getId()) - -opObj11 = procUnitConf1.addOperation(name='ProfileSelector', optype='other') -opObj11.addParameter(name='profileRangeList', value='120,183', format='intlist') - -opObj11 = procUnitConf1.addOperation(name='RTIPlot', optype='other') -opObj11.addParameter(name='wintitle', value='Jicamarca Radio Observatory', format='str') -opObj11.addParameter(name='showprofile', value='0', format='int') -opObj11.addParameter(name='xmin', value='0', format='int') -opObj11.addParameter(name='xmax', value='24', format='int') -opObj11.addParameter(name='figpath', value="/home/nanosat/schain/schainpy/scripts/figs", format='str') -opObj11.addParameter(name='wr_period', value='5', format='int') -opObj11.addParameter(name='exp_code', value='22', format='int') - - -controller.start() diff --git a/schainpy/scripts/protocol, b/schainpy/scripts/protocol, deleted file mode 100644 index e69de29..0000000 --- a/schainpy/scripts/protocol, +++ /dev/null diff --git a/schainpy/scripts/receiver.py b/schainpy/scripts/receiver.py deleted file mode 100644 index c06b1ce..0000000 --- a/schainpy/scripts/receiver.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -''' -Created on Jul 7, 2014 - -@author: roj-idl71 -''' -import os, sys - -from schainpy.controller import Project - -if __name__ == '__main__': - desc = "Segundo Test" - - controllerObj = Project() - controllerObj.setup(id='191', name='test01', description=desc) - - proc1 = controllerObj.addProcUnit(name='ReceiverData') - proc1.addParameter(name='realtime', value='0', format='bool') - proc1.addParameter(name='plottypes', value='snr', format='str') - proc1.addParameter(name='throttle', value='5', format='int') - proc1.addParameter(name='interactive', value='0', format='bool') - # proc1.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str') - ## TODO Agregar direccion de server de publicacion a graficos como variable - op3 = proc1.addOperation(name='PlotSNRData', optype='other') - op3.addParameter(name='wintitle', value='Julia 150Km', format='str') - op3.addParameter(name='zmax', value='30', format='int') - op3.addParameter(name='zmin', value='-10', format='int') - op3.addParameter(name='save', value='/home/nanosat/Pictures', format='str') - op3.addParameter(name='show', value='0', format='bool') - # # - # op4 = proc1.addOperation(name='PlotDOPData', optype='other') - # op4.addParameter(name='wintitle', value='Julia 150Km', format='str') - # op4.addParameter(name='save', value='/home/nanosat/Pictures', format='str') - # op4.addParameter(name='show', value='0', format='bool') - # op4.addParameter(name='colormap', value='jet', format='str') - - # op1 = proc1.addOperation(name='PlotRTIData', optype='other') - # op1.addParameter(name='wintitle', value='Julia 150Km', format='str') - # op1.addParameter(name='save', value='/home/nanosat/Pictures', format='str') - # op1.addParameter(name='show', value='0', format='bool') - # op1.addParameter(name='colormap', value='jet', format='str') - # # - # op2 = proc1.addOperation(name='PlotCOHData', optype='other') - # op2.addParameter(name='wintitle', value='Julia 150Km', format='str') - # op2.addParameter(name='save', value='/home/nanosat/Pictures', format='str') - # op2.addParameter(name='colormap', value='jet', format='str') - # op2.addParameter(name='show', value='0', format='bool') - # # # # - # op6 = proc1.addOperation(name='PlotPHASEData', optype='other') - # op6.addParameter(name='wintitle', value='Julia 150Km', format='str') - # op6.addParameter(name='save', value='/home/nanosat/Pictures', format='str') - # op6.addParameter(name='show', value='1', format='bool') - # # - # # proc2 = controllerObj.addProcUnit(name='ReceiverData') - # # proc2.addParameter(name='server', value='juanca', format='str') - # # proc2.addParameter(name='plottypes', value='snr,dop', format='str') - # # - - - - controllerObj.start() diff --git a/schainpy/scripts/schain.xml b/schainpy/scripts/schain.xml index 83d2ad5..03d348c 100644 --- a/schainpy/scripts/schain.xml +++ b/schainpy/scripts/schain.xml @@ -1 +1,5 @@ - \ No newline at end of file +<<<<<<< HEAD + +======= + +>>>>>>> v2.3 diff --git a/schainpy/utils/.desktop b/schainpy/utils/.desktop new file mode 100644 index 0000000..b1e940f --- /dev/null +++ b/schainpy/utils/.desktop @@ -0,0 +1,6 @@ +[Desktop Entry] +Encoding=UTF-8 +Name=Link to +Type=Link +URL=file:///home/nanosat/schain/schainpy/utils/parameters.txt +Icon=text-plain diff --git a/schainpy/utils/log.py b/schainpy/utils/log.py index 85f1489..5a08497 100644 --- a/schainpy/utils/log.py +++ b/schainpy/utils/log.py @@ -1,4 +1,4 @@ -""". +""" SCHAINPY - LOG Simple helper for log standarization Usage: @@ -14,32 +14,46 @@ SCHAINPY - LOG [NEVER GONNA] - give you up with color red as background and white as foreground. """ - +import os +import sys import click - def warning(message): click.echo(click.style('[WARNING] - ' + message, fg='yellow')) - pass def error(message): - click.echo(click.style('[ERROR] - ' + message, fg='red')) - pass + click.echo(click.style('[ERROR] - ' + message, fg='red', bg='black')) def success(message): click.echo(click.style(message, fg='green')) - pass -def log(message): - click.echo('[LOG] - ' + message) - pass - +def log(message, topic='LOG'): + click.echo('[{}] - {}'.format(topic, message)) def makelogger(topic, bg='reset', fg='reset'): def func(message): click.echo(click.style('[{}] - '.format(topic.upper()) + message, bg=bg, fg=fg)) return func + +class LoggerForFile(): + def __init__(self, filename): + self.old_stdout=sys.stdout + cwd = os.getcwd() + self.log_file = open(os.path.join(cwd, filename), 'w+') + def write(self, text): + text = text.rstrip() + if not text: + return + self.log_file.write(text + '\n') + self.old_stdout.write(text + '\n') + def flush(self): + self.old_stdout.flush() + +def logToFile(filename='log.log'): + logger = LoggerForFile(filename) + sys.stdout = logger + diff --git a/schainpy/utils/paramsFinder.py b/schainpy/utils/paramsFinder.py new file mode 100644 index 0000000..3361134 --- /dev/null +++ b/schainpy/utils/paramsFinder.py @@ -0,0 +1,81 @@ +import schainpy +from schainpy.model import Operation, ProcessingUnit +from importlib import import_module +from pydoc import locate + +def clean_modules(module): + noEndsUnder = [x for x in module if not x.endswith('__')] + noStartUnder = [x for x in noEndsUnder if not x.startswith('__')] + noFullUpper = [x for x in noStartUnder if not x.isupper()] + return noFullUpper + +def check_module(possible, instance): + def check(x): + try: + instancia = locate('schainpy.model.{}'.format(x)) + return isinstance(instancia(), instance) + except Exception as e: + return False + clean = clean_modules(possible) + return [x for x in clean if check(x)] + + +def getProcs(): + module = dir(import_module('schainpy.model')) + procs = check_module(module, ProcessingUnit) + try: + procs.remove('ProcessingUnit') + except Exception as e: + pass + return procs + +def getOperations(): + module = dir(import_module('schainpy.model')) + noProcs = [x for x in module if not x.endswith('Proc')] + operations = check_module(noProcs, Operation) + try: + operations.remove('Operation') + except Exception as e: + pass + return operations + +def getArgs(op): + module = locate('schainpy.model.{}'.format(op)) + args = module().getAllowedArgs() + try: + args.remove('self') + except Exception as e: + pass + try: + args.remove('dataOut') + except Exception as e: + pass + return args + +def getAll(): + allModules = dir(import_module('schainpy.model')) + modules = check_module(allModules, Operation) + modules.extend(check_module(allModules, ProcessingUnit)) + return modules + +def formatArgs(op): + args = getArgs(op) + + argsAsKey = ["\t'{}'".format(x) for x in args] + argsFormatted = ": 'string',\n".join(argsAsKey) + + print op + print "parameters = { \n" + argsFormatted + ": 'string',\n }" + print '\n' + + +if __name__ == "__main__": + getAll() + [formatArgs(x) for x in getAll()] + + ''' + parameters = { + 'id': , + 'wintitle': , + } + ''' \ No newline at end of file diff --git a/schainpy/trash b/schainpy/utils/trash similarity index 100% rename from schainpy/trash rename to schainpy/utils/trash diff --git a/setup.py b/setup.py index 61b0f7a..be416d7 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,8 @@ -""". - +''' Created on Jul 16, 2014 @author: Miguel Urco -""" +''' from setuptools import setup, Extension from setuptools.command.build_ext import build_ext as _build_ext @@ -19,43 +18,52 @@ class build_ext(_build_ext): setup(name="schainpy", - version=__version__, - description="Python tools to read, write and process Jicamarca data", - author="Miguel Urco", - author_email="miguel.urco@jro.igp.gob.pe", - url="http://jro.igp.gob.pe", - packages = {'schainpy', - 'schainpy.model', - 'schainpy.model.data', - 'schainpy.model.graphics', - 'schainpy.model.io', - 'schainpy.model.proc', - 'schainpy.model.serializer', - 'schainpy.model.utils', - 'schainpy.gui', - 'schainpy.gui.figures', - 'schainpy.gui.viewcontroller', - 'schainpy.gui.viewer', - 'schainpy.gui.viewer.windows'}, - ext_package='schainpy', - py_modules=[''], - package_data={'': ['schain.conf.template'], - 'schainpy.gui.figures': ['*.png','*.jpg'], - }, - include_package_data=False, - scripts =['schainpy/gui/schainGUI', - 'schainpy/scripts/schain'], - ext_modules=[ - Extension("cSchain", ["schainpy/model/proc/extensions.c"] - )], - cmdclass={'build_ext':build_ext}, - setup_requires=["numpy >= 1.11.2"], - install_requires=[ + version=__version__, + description="Python tools to read, write and process Jicamarca data", + author="Miguel Urco", + author_email="miguel.urco@jro.igp.gob.pe", + url="http://jro.igp.gob.pe", + packages = {'schainpy', + 'schainpy.model', + 'schainpy.model.data', + 'schainpy.model.graphics', + 'schainpy.model.io', + 'schainpy.model.proc', + 'schainpy.model.serializer', + 'schainpy.model.utils', + 'schainpy.gui', + 'schainpy.gui.figures', + 'schainpy.gui.viewcontroller', + 'schainpy.gui.viewer', + 'schainpy.gui.viewer.windows'}, + ext_package='schainpy', + py_modules=[''], + package_data={'': ['schain.conf.template'], + 'schainpy.gui.figures': ['*.png','*.jpg'], + }, + include_package_data=False, + scripts =['schainpy/gui/schainGUI'], + ext_modules=[ + Extension("cSchain", ["schainpy/model/proc/extensions.c"] + )], + entry_points={ + 'console_scripts': [ + 'schain = schaincli.cli:main', + ], + }, + cmdclass={'build_ext':build_ext}, + setup_requires=["numpy >= 1.11.2"], + install_requires=[ "scipy >= 0.14.0", "h5py >= 2.2.1", "matplotlib >= 1.4.2", - "pyfits >= 3.4", + "pyfits >= 3.4", + "paramiko >= 2.1.2", "paho-mqtt >= 1.2", "zmq", - ], - ) + "fuzzywuzzy", + "click", + "colorama", + "python-Levenshtein" + ], + ) diff --git a/trash b/trash deleted file mode 100644 index 384299d..0000000 --- a/trash +++ /dev/null @@ -1 +0,0 @@ -You should install "digital_rf_hdf5" module if you want to read USRP data