##// END OF EJS Templates
schain
RSS
fixing merge
Juan C. Espinoza -
r938:9e80eea2eb45 merge
parent child
Show More
Show file before | Show file after | Hide comments
@@ -8,6 +8,7 import ast
8 import datetime
8 import datetime
9 import traceback
9 import traceback
10 import math
10 import math
11 import time
11 from multiprocessing import Process, Queue, cpu_count
12 from multiprocessing import Process, Queue, cpu_count
12
13
13 import schainpy
14 import schainpy
@@ -66,8 +67,7 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_da
66 for process in processes:
67 for process in processes:
67 process.join()
68 process.join()
68 process.terminate()
69 process.terminate()
69 sys.exit()
70 time.sleep(3)
70
71
71
72 class ParameterConf():
72 class ParameterConf():
73
73
Show file before | Show file after | Hide comments
@@ -65,7 +65,7 def hildebrand_sekhon(data, navg):
65 anoise : noise's level
65 anoise : noise's level
66 """
66 """
67
67
68 sortdata = numpy.sort(data,axis=None)
68 sortdata = numpy.sort(data, axis=None)
69 # lenOfData = len(sortdata)
69 # lenOfData = len(sortdata)
70 # nums_min = lenOfData*0.2
70 # nums_min = lenOfData*0.2
71 #
71 #
Show file before | Show file after | Hide comments
@@ -24,9 +24,8 class PlotData(Operation, Process):
24
24
25 CODE = 'Figure'
25 CODE = 'Figure'
26 colormap = 'jro'
26 colormap = 'jro'
27 CONFLATE = True
27 CONFLATE = False
28 __MAXNUMX = 80
28 __MAXNUMX = 80
29 __MAXNUMY = 80
30 __missing = 1E30
29 __missing = 1E30
31
30
32 def __init__(self, **kwargs):
31 def __init__(self, **kwargs):
@@ -55,7 +54,9 class PlotData(Operation, Process):
55 self.xrange = kwargs.get('xrange', 24)
54 self.xrange = kwargs.get('xrange', 24)
56 self.ymin = kwargs.get('ymin', None)
55 self.ymin = kwargs.get('ymin', None)
57 self.ymax = kwargs.get('ymax', None)
56 self.ymax = kwargs.get('ymax', None)
57 self.__MAXNUMY = kwargs.get('decimation', 80)
58 self.throttle_value = 5
58 self.throttle_value = 5
59 self.times = []
59
60
60 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
61 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
61
62
@@ -120,15 +121,23 class PlotData(Operation, Process):
120 receiver = context.socket(zmq.SUB)
121 receiver = context.socket(zmq.SUB)
121 receiver.setsockopt(zmq.SUBSCRIBE, '')
122 receiver.setsockopt(zmq.SUBSCRIBE, '')
122 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
123 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
124
123 if 'server' in self.kwargs['parent']:
125 if 'server' in self.kwargs['parent']:
124 receiver.connect('ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
126 receiver.connect('ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
125 else:
127 else:
126 receiver.connect("ipc:///tmp/zmq.plots")
128 receiver.connect("ipc:///tmp/zmq.plots")
127
129
130 seconds_passed = 0
131
128 while True:
132 while True:
129 try:
133 try:
130 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
134 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)#flags=zmq.NOBLOCK
135 self.started = self.data['STARTED']
131 self.dataOut = self.data['dataOut']
136 self.dataOut = self.data['dataOut']
137
138 if (len(self.times) < len(self.data['times']) and not self.started and self.data['ENDED']):
139 continue
140
132 self.times = self.data['times']
141 self.times = self.data['times']
133 self.times.sort()
142 self.times.sort()
134 self.throttle_value = self.data['throttle']
143 self.throttle_value = self.data['throttle']
@@ -136,16 +145,25 class PlotData(Operation, Process):
136 self.max_time = self.times[-1]
145 self.max_time = self.times[-1]
137
146
138 if self.isConfig is False:
147 if self.isConfig is False:
148 print 'setting up'
139 self.setup()
149 self.setup()
140 self.isConfig = True
150 self.isConfig = True
141 self.__plot()
151 self.__plot()
142
152
143 if self.data['ENDED'] is True:
153 if self.data['ENDED'] is True:
154 print '********GRAPHIC ENDED********'
155 self.ended = True
144 self.isConfig = False
156 self.isConfig = False
157 self.__plot()
158 elif seconds_passed >= self.data['throttle']:
159 print 'passed', seconds_passed
160 self.__plot()
161 seconds_passed = 0
145
162
146 except zmq.Again as e:
163 except zmq.Again as e:
147 print 'Waiting for data...'
164 print 'Waiting for data...'
148 plt.pause(self.throttle_value)
165 plt.pause(2)
166 seconds_passed += 2
149
167
150 def close(self):
168 def close(self):
151 if self.dataOut:
169 if self.dataOut:
@@ -472,15 +490,14 class PlotRTIData(PlotData):
472
490
473 self.z = np.array(self.z)
491 self.z = np.array(self.z)
474 for n, ax in enumerate(self.axes):
492 for n, ax in enumerate(self.axes):
475
476 x, y, z = self.fill_gaps(*self.decimate())
493 x, y, z = self.fill_gaps(*self.decimate())
477 xmin = self.min_time
494 xmin = self.min_time
478 xmax = xmin+self.xrange*60*60
495 xmax = xmin+self.xrange*60*60
496 self.zmin = self.zmin if self.zmin else np.min(self.z)
497 self.zmax = self.zmax if self.zmax else np.max(self.z)
479 if ax.firsttime:
498 if ax.firsttime:
480 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
499 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
481 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
500 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
482 self.zmin = self.zmin if self.zmin else np.nanmin(self.z)
483 self.zmax = self.zmax if self.zmax else np.nanmax(self.z)
484 plot = ax.pcolormesh(x, y, z[n].T,
501 plot = ax.pcolormesh(x, y, z[n].T,
485 vmin=self.zmin,
502 vmin=self.zmin,
486 vmax=self.zmax,
503 vmax=self.zmax,
Show file before | Show file after | Hide comments
@@ -1,7 +1,7
1 import os
1 import os
2 import datetime
2 import datetime
3 import numpy
3 import numpy
4
4 import inspect
5 from figure import Figure, isRealtime, isTimeInHourRange
5 from figure import Figure, isRealtime, isTimeInHourRange
6 from plotting_codes import *
6 from plotting_codes import *
7
7
@@ -642,7 +642,6 class ParametersPlot(Figure):
642 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
642 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
643 server=None, folder=None, username=None, password=None,
643 server=None, folder=None, username=None, password=None,
644 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
644 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
645
646 """
645 """
647
646
648 Input:
647 Input:
@@ -792,7 +791,7 class Parameters1Plot(Figure):
792 self.__nsubplots = 1
791 self.__nsubplots = 1
793
792
794 self.WIDTH = 800
793 self.WIDTH = 800
795 self.HEIGHT = 150
794 self.HEIGHT = 180
796 self.WIDTHPROF = 120
795 self.WIDTHPROF = 120
797 self.HEIGHTPROF = 0
796 self.HEIGHTPROF = 0
798 self.counter_imagwr = 0
797 self.counter_imagwr = 0
@@ -857,7 +856,7 class Parameters1Plot(Figure):
857 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
856 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
858 server=None, folder=None, username=None, password=None,
857 server=None, folder=None, username=None, password=None,
859 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
858 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
860
859 #print inspect.getargspec(self.run).args
861 """
860 """
862
861
863 Input:
862 Input:
Show file before | Show file after | Hide comments
@@ -888,7 +888,8 class JRODataReader(JRODataIO):
888 print '[Reading] No more files to read'
888 print '[Reading] No more files to read'
889 return 0
889 return 0
890
890
891 print '[Reading] Setting the file: %s' % self.filename
891 if self.verbose:
892 print '[Reading] Setting the file: %s' % self.filename
892
893
893 self.__readFirstHeader()
894 self.__readFirstHeader()
894 self.nReadBlocks = 0
895 self.nReadBlocks = 0
@@ -1053,9 +1054,10 class JRODataReader(JRODataIO):
1053
1054
1054 break
1055 break
1055
1056
1056 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1057 if self.verbose:
1057 self.processingHeaderObj.dataBlocksPerFile,
1058 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1058 self.dataOut.datatime.ctime())
1059 self.processingHeaderObj.dataBlocksPerFile,
1060 self.dataOut.datatime.ctime())
1059 return 1
1061 return 1
1060
1062
1061 def __readFirstHeader(self):
1063 def __readFirstHeader(self):
@@ -1266,7 +1268,9 class JRODataReader(JRODataIO):
1266 blocktime=None,
1268 blocktime=None,
1267 queue=None,
1269 queue=None,
1268 skip=None,
1270 skip=None,
1269 cursor=None):
1271 cursor=None,
1272 warnings=True,
1273 verbose=True):
1270
1274
1271 if path == None:
1275 if path == None:
1272 raise ValueError, "[Reading] The path is not valid"
1276 raise ValueError, "[Reading] The path is not valid"
@@ -1338,6 +1342,9 class JRODataReader(JRODataIO):
1338 self.selBlocksize = blocksize
1342 self.selBlocksize = blocksize
1339 self.selBlocktime = blocktime
1343 self.selBlocktime = blocktime
1340
1344
1345 # Verbose-----------
1346 self.verbose = verbose
1347 self.warnings = warnings
1341
1348
1342 if not(self.setNextFile()):
1349 if not(self.setNextFile()):
1343 if (startDate!=None) and (endDate!=None):
1350 if (startDate!=None) and (endDate!=None):
Show file before | Show file after | Hide comments
@@ -3,6 +3,29
3 $Author: murco $
3 $Author: murco $
4 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
4 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
5 '''
5 '''
6 import inspect
7 from fuzzywuzzy import process
8
9 def checkKwargs(method, kwargs):
10 currentKwargs = kwargs
11 choices = inspect.getargspec(method).args
12 try:
13 choices.remove('self')
14 except Exception as e:
15 pass
16
17 try:
18 choices.remove('dataOut')
19 except Exception as e:
20 pass
21
22 for kwarg in kwargs:
23 fuzz = process.extractOne(kwarg, choices)
24 if fuzz is None:
25 continue
26 if fuzz[1] < 100:
27 raise Exception('\x1b[0;32;40mDid you mean {} instead of {} in {}? \x1b[0m'.
28 format(fuzz[0], kwarg, method.__self__.__class__.__name__))
6
29
7 class ProcessingUnit(object):
30 class ProcessingUnit(object):
8
31
@@ -41,14 +64,18 class ProcessingUnit(object):
41
64
42 self.args = args
65 self.args = args
43 self.kwargs = kwargs
66 self.kwargs = kwargs
67 checkKwargs(self.run, kwargs)
68
69 def getAllowedArgs(self):
70 return inspect.getargspec(self.run).args
44
71
45 def addOperationKwargs(self, objId, **kwargs):
72 def addOperationKwargs(self, objId, **kwargs):
46 '''
73 '''
47 '''
74 '''
48
75
49 self.operationKwargs[objId] = kwargs
76 self.operationKwargs[objId] = kwargs
50
77
51
78
52 def addOperation(self, opObj, objId):
79 def addOperation(self, opObj, objId):
53
80
54 """
81 """
@@ -117,17 +144,17 class ProcessingUnit(object):
117 #Executing the self method
144 #Executing the self method
118
145
119 if hasattr(self, 'mp'):
146 if hasattr(self, 'mp'):
120 if name=='run':
147 if name=='run':
121 if self.mp is False:
148 if self.mp is False:
122 self.mp = True
149 self.mp = True
123 self.start()
150 self.start()
124 else:
151 else:
125 self.operationKwargs[opId]['parent'] = self.kwargs
152 self.operationKwargs[opId]['parent'] = self.kwargs
126 methodToCall(**self.operationKwargs[opId])
153 methodToCall(**self.operationKwargs[opId])
127 else:
154 else:
128 if name=='run':
155 if name=='run':
129 methodToCall(**self.kwargs)
156 methodToCall(**self.kwargs)
130 else:
157 else:
131 methodToCall(**self.operationKwargs[opId])
158 methodToCall(**self.operationKwargs[opId])
132
159
133 if self.dataOut is None:
160 if self.dataOut is None:
@@ -168,7 +195,7 class ProcessingUnit(object):
168 else:
195 else:
169 externalProcObj.run(self.dataOut, **externalProcObj.kwargs)
196 externalProcObj.run(self.dataOut, **externalProcObj.kwargs)
170 self.operationKwargs[objId] = externalProcObj.kwargs
197 self.operationKwargs[objId] = externalProcObj.kwargs
171
198
172
199
173 return True
200 return True
174
201
@@ -283,6 +310,10 class Operation(object):
283 self.__buffer = None
310 self.__buffer = None
284 self.isConfig = False
311 self.isConfig = False
285 self.kwargs = kwargs
312 self.kwargs = kwargs
313 checkKwargs(self.run, kwargs)
314
315 def getAllowedArgs(self):
316 return inspect.getargspec(self.run).args
286
317
287 def setup(self):
318 def setup(self):
288
319
Show file before | Show file after | Hide comments
@@ -7,7 +7,7 import json
7 import numpy
7 import numpy
8 import paho.mqtt.client as mqtt
8 import paho.mqtt.client as mqtt
9 import zmq
9 import zmq
10 import cPickle as pickle
10 from profilehooks import profile
11 import datetime
11 import datetime
12 from zmq.utils.monitor import recv_monitor_message
12 from zmq.utils.monitor import recv_monitor_message
13 from functools import wraps
13 from functools import wraps
@@ -29,7 +29,7 def roundFloats(obj):
29 elif isinstance(obj, float):
29 elif isinstance(obj, float):
30 return round(obj, 2)
30 return round(obj, 2)
31
31
32 def decimate(z):
32 def decimate(z, MAXNUMY):
33 # dx = int(len(self.x)/self.__MAXNUMX) + 1
33 # dx = int(len(self.x)/self.__MAXNUMX) + 1
34
34
35 dy = int(len(z[0])/MAXNUMY) + 1
35 dy = int(len(z[0])/MAXNUMY) + 1
@@ -107,7 +107,7 class PublishData(Operation):
107 print "MQTT Conection error."
107 print "MQTT Conection error."
108 self.client = False
108 self.client = False
109
109
110 def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, **kwargs):
110 def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, verbose=True, **kwargs):
111 self.counter = 0
111 self.counter = 0
112 self.topic = kwargs.get('topic', 'schain')
112 self.topic = kwargs.get('topic', 'schain')
113 self.delay = kwargs.get('delay', 0)
113 self.delay = kwargs.get('delay', 0)
@@ -119,6 +119,8 class PublishData(Operation):
119 self.zeromq = zeromq
119 self.zeromq = zeromq
120 self.mqtt = kwargs.get('plottype', 0)
120 self.mqtt = kwargs.get('plottype', 0)
121 self.client = None
121 self.client = None
122 self.verbose = verbose
123 self.dataOut.firstdata = True
122 setup = []
124 setup = []
123 if mqtt is 1:
125 if mqtt is 1:
124 self.client = mqtt.Client(
126 self.client = mqtt.Client(
@@ -150,6 +152,7 class PublishData(Operation):
150 self.zmq_socket.connect(address)
152 self.zmq_socket.connect(address)
151 time.sleep(1)
153 time.sleep(1)
152
154
155
153 def publish_data(self):
156 def publish_data(self):
154 self.dataOut.finished = False
157 self.dataOut.finished = False
155 if self.mqtt is 1:
158 if self.mqtt is 1:
@@ -230,8 +233,11 class PublishData(Operation):
230 self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
233 self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
231
234
232 if self.zeromq is 1:
235 if self.zeromq is 1:
233 print '[Sending] {} - {}'.format(self.dataOut.type, self.dataOut.datatime)
236 if self.verbose:
237 print '[Sending] {} - {}'.format(self.dataOut.type, self.dataOut.datatime)
234 self.zmq_socket.send_pyobj(self.dataOut)
238 self.zmq_socket.send_pyobj(self.dataOut)
239 self.dataOut.firstdata = False
240
235
241
236 def run(self, dataOut, **kwargs):
242 def run(self, dataOut, **kwargs):
237 self.dataOut = dataOut
243 self.dataOut = dataOut
@@ -246,12 +252,11 class PublishData(Operation):
246 if self.zeromq is 1:
252 if self.zeromq is 1:
247 self.dataOut.finished = True
253 self.dataOut.finished = True
248 self.zmq_socket.send_pyobj(self.dataOut)
254 self.zmq_socket.send_pyobj(self.dataOut)
249
255 self.zmq_socket.close()
250 if self.client:
256 if self.client:
251 self.client.loop_stop()
257 self.client.loop_stop()
252 self.client.disconnect()
258 self.client.disconnect()
253
259
254
255 class ReceiverData(ProcessingUnit, Process):
260 class ReceiverData(ProcessingUnit, Process):
256
261
257 throttle_value = 5
262 throttle_value = 5
@@ -330,12 +335,13 class ReceiverData(ProcessingUnit, Process):
330
335
331 return sendDataThrottled
336 return sendDataThrottled
332
337
338
333 def send(self, data):
339 def send(self, data):
334 # print '[sending] data=%s size=%s' % (data.keys(), len(data['times']))
340 # print '[sending] data=%s size=%s' % (data.keys(), len(data['times']))
335 self.sender.send_pyobj(data)
341 self.sender.send_pyobj(data)
336
342
337 def update(self):
338
343
344 def update(self):
339 t = self.dataOut.utctime
345 t = self.dataOut.utctime
340
346
341 if t in self.data['times']:
347 if t in self.data['times']:
@@ -398,11 +404,14 class ReceiverData(ProcessingUnit, Process):
398 self.sender_web = self.context.socket(zmq.PUB)
404 self.sender_web = self.context.socket(zmq.PUB)
399 self.sender_web.connect(self.plot_address)
405 self.sender_web.connect(self.plot_address)
400 time.sleep(1)
406 time.sleep(1)
407
401 if 'server' in self.kwargs:
408 if 'server' in self.kwargs:
402 self.sender.bind("ipc:///tmp/{}.plots".format(self.kwargs['server']))
409 self.sender.bind("ipc:///tmp/{}.plots".format(self.kwargs['server']))
403 else:
410 else:
404 self.sender.bind("ipc:///tmp/zmq.plots")
411 self.sender.bind("ipc:///tmp/zmq.plots")
405
412
413 time.sleep(3)
414
406 t = Thread(target=self.event_monitor, args=(monitor,))
415 t = Thread(target=self.event_monitor, args=(monitor,))
407 t.start()
416 t.start()
408
417
@@ -413,6 +422,10 class ReceiverData(ProcessingUnit, Process):
413
422
414 self.update()
423 self.update()
415
424
425 if self.dataOut.firstdata is True:
426 self.data['STARTED'] = True
427
428
416 if self.dataOut.finished is True:
429 if self.dataOut.finished is True:
417 self.send(self.data)
430 self.send(self.data)
418 self.connections -= 1
431 self.connections -= 1
@@ -421,6 +434,7 class ReceiverData(ProcessingUnit, Process):
421 self.data['ENDED'] = True
434 self.data['ENDED'] = True
422 self.send(self.data)
435 self.send(self.data)
423 self.setup()
436 self.setup()
437 self.started = False
424 else:
438 else:
425 if self.realtime:
439 if self.realtime:
426 self.send(self.data)
440 self.send(self.data)
@@ -429,6 +443,7 class ReceiverData(ProcessingUnit, Process):
429 self.sendData(self.send, self.data)
443 self.sendData(self.send, self.data)
430 self.started = True
444 self.started = True
431
445
446 self.data['STARTED'] = False
432 return
447 return
433
448
434 def sendToWeb(self):
449 def sendToWeb(self):
Show file before | Show file after | Hide comments
@@ -11,26 +11,26 def fiber(cursor, skip, q, dt):
11 controllerObj.setup(id='191', name='test01', description=desc)
11 controllerObj.setup(id='191', name='test01', description=desc)
12
12
13 readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
13 readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
14 path='/home/nanosat/data/hysell_data20/pdata',
14 path='/home/nanosat/data/sp1_f0',
15 startDate=dt,
15 startDate=dt,
16 endDate=dt,
16 endDate=dt,
17 startTime="00:00:00",
17 startTime="00:00:00",
18 endTime="23:59:59",
18 endTime="23:59:59",
19 online=0,
19 online=0,
20 #set=1426485881,
20 #set=1426485881,
21 delay=10,
22 walk=1,
21 walk=1,
23 queue=q,
22 queue=q,
24 cursor=cursor,
23 cursor=cursor,
25 skip=skip,
24 skip=skip,
25 verbose=1
26 #timezone=-5*3600
26 #timezone=-5*3600
27 )
27 )
28
28
29 # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
29 # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
30 #
30 #
31 procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
31 procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
32 # opObj11 = procUnitConfObj2.addParameter(name='pairsList', value='(0,1)', format='pairslist')
32 # procUnitConfObj2.addParameter(name='nipp', value='5', format='int')
33 #
33
34 procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
34 procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
35 opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
35 opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
36
36
@@ -44,10 +44,19 def fiber(cursor, skip, q, dt):
44 # opObj11.addParameter(name='save', value='1', format='int')
44 # opObj11.addParameter(name='save', value='1', format='int')
45 # opObj11.addParameter(name='figpath', value=figpath, format='str')
45 # opObj11.addParameter(name='figpath', value=figpath, format='str')
46
46
47 # opObj11 = procUnitConfObj2.addOperation(name='RTIPlot', optype='other')
47 # opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
48 # opObj11.addParameter(name='channelList', value='0', format='intList')
49
48 # opObj11.addParameter(name='id', value='2000', format='int')
50 # opObj11.addParameter(name='id', value='2000', format='int')
49 # opObj11.addParameter(name='wintitzmaxle', value='HF_Jicamarca', format='str')
51 # # opObj11.addParameter(name='colormap', value='0', format='bool')
50 # opObj11.addParameter(name='showprofile', value='0', format='int')
52 # opObj11.addParameter(name='onlySNR', value='1', format='bool')
53 # opObj11.addParameter(name='DOP', value='0', format='bool')
54 # # opObj11.addParameter(name='showSNR', value='1', format='bool')
55 # # opObj11.addParameter(name='SNRthresh', value='0', format='int')
56 # opObj11.addParameter(name='SNRmin', value='-10', format='int')
57 # opObj11.addParameter(name='SNRmax', value='30', format='int')
58
59 # opObj11.addParameter(name='showSNR', value='1', format='int')
51 # # opObj11.addParameter(name='channelList', value='0', format='intlist')
60 # # opObj11.addParameter(name='channelList', value='0', format='intlist')
52 # # opObj11.addParameter(name='xmin', value='0', format='float')
61 # # opObj11.addParameter(name='xmin', value='0', format='float')
53 # opObj11.addParameter(name='xmin', value='0', format='float')
62 # opObj11.addParameter(name='xmin', value='0', format='float')
@@ -60,13 +69,14 def fiber(cursor, skip, q, dt):
60 #
69 #
61 opObj12 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
70 opObj12 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
62 opObj12.addParameter(name='zeromq', value=1, format='int')
71 opObj12.addParameter(name='zeromq', value=1, format='int')
72 opObj12.addParameter(name='verbose', value=0, format='bool')
63
73
64
74
65 # opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
75 # opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
66 # opObj13.addParameter(name='zeromq', value=1, format='int')
76 # opObj13.addParameter(name='zeromq', value=1, format='int')
67 # opObj13.addParameter(name='server', value="juanca", format='str')
77 # opObj13.addParameter(name='server', value="juanca", format='str')
68
78
69 opObj12.addParameter(name='delay', value=1, format='int')
79 opObj12.addParameter(name='delay', value=0, format='int')
70
80
71
81
72 # print "Escribiendo el archivo XML"
82 # print "Escribiendo el archivo XML"
@@ -84,4 +94,4 if __name__ == '__main__':
84 parser = argparse.ArgumentParser(description='Set number of parallel processes')
94 parser = argparse.ArgumentParser(description='Set number of parallel processes')
85 parser.add_argument('--nProcess', default=1, type=int)
95 parser.add_argument('--nProcess', default=1, type=int)
86 args = parser.parse_args()
96 args = parser.parse_args()
87 multiSchain(fiber, nProcess=args.nProcess, startDate='2015/09/26', endDate='2015/09/26')
97 multiSchain(fiber, nProcess=args.nProcess, startDate='2017/01/26', endDate='2017/01/28')
Show file before | Show file after | Hide comments
@@ -1,75 +1,97
1 #!/usr/bin/env python
1 import argparse
2 '''
3 Created on Jul 7, 2014
4
2
5 @author: roj-idl71
3 from schainpy.controller import Project, multiSchain
6 '''
7 import os, sys
8 from datetime import datetime, timedelta
9 import multiprocessing
10 from schainpy.controller import Project
11
4
12 def main(date):
5 desc = "HF_EXAMPLE"
13
6
14 controllerObj = Project()
7 def fiber(cursor, skip, q, dt):
15
16 controllerObj.setup(id='191', name='test01', description='')
17
18 readUnitConfObj = controllerObj.addReadUnit(datatype='Spectra',
19 path='/home/nanosat/data/zeus',
20 startDate=date,
21 endDate=date,
22 startTime='00:00:00',
23 endTime='23:59:59',
24 online=0,
25 walk=1,
26 expLabel='')
27
28 procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
29 #opObj11 = procUnitConfObj1.addOperation(name='removeDC')
30 #opObj11.addParameter(name='mode', value='1', format='int')
31
32 #opObj11 = procUnitConfObj1.addOperation(name='removeInterference')
33
34
35 opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
36 opObj11.addParameter(name='id', value='10', format='int')
37 opObj11.addParameter(name='wintitle', value='150Km', format='str')
38 opObj11.addParameter(name='colormap', value='jro', format='str')
39 opObj11.addParameter(name='xaxis', value='time', format='str')
40 opObj11.addParameter(name='xmin', value='0', format='int')
41 opObj11.addParameter(name='xmax', value='23', format='int')
42 #opObj11.addParameter(name='ymin', value='100', format='int')
43 #opObj11.addParameter(name='ymax', value='150', format='int')
44 opObj11.addParameter(name='zmin', value='10', format='int')
45 opObj11.addParameter(name='zmax', value='35', format='int')
46
8
9 controllerObj = Project()
47
10
48
11 controllerObj.setup(id='191', name='test01', description=desc)
49
12
50 opObject12 = procUnitConfObj1.addOperation(name='PlotRTIData', optype='other')
13 readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
51 opObject12.addParameter(name='id', value='12', format='int')
14 path='/home/nanosat/data/julia',
52 opObject12.addParameter(name='wintitle', value='150Km', format='str')
15 startDate=dt,
53 opObject12.addParameter(name='colormap', value='jro', format='str')
16 endDate=dt,
54 opObject12.addParameter(name='xaxis', value='time', format='str')
17 startTime="00:00:00",
55 opObject12.addParameter(name='xmin', value='0', format='int')
18 endTime="23:59:59",
56 opObject12.addParameter(name='xmax', value='23', format='int')
19 online=0,
57 #opObject12.addParameter(name='ymin', value='100', format='int')
20 #set=1426485881,
58 #opObject12.addParameter(name='ymax', value='150', format='int')
21 delay=10,
59 opObject12.addParameter(name='zmin', value='10', format='int')
22 walk=1,
60 opObject12.addParameter(name='zmax', value='35', format='int')
23 queue=q,
61 #opObject12.addParameter(name='pause', value='1', format='bool')
24 cursor=cursor,
62 opObject12.addParameter(name='show', value='0', format='bool')
25 skip=skip,
63 opObject12.addParameter(name='save', value='/tmp', format='str')
26 #timezone=-5*3600
64
27 )
28
29 # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
30 #
31 procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
32 # procUnitConfObj2.addParameter(name='nipp', value='5', format='int')
33
34 # procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
35 # opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
36
37 #
38 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
39 # opObj11.addParameter(name='id', value='1000', format='int')
40 # opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
41 # opObj11.addParameter(name='channelList', value='0', format='intlist')
42 # opObj11.addParameter(name='zmin', value='-120', format='float')
43 # opObj11.addParameter(name='zmax', value='-70', format='float')
44 # opObj11.addParameter(name='save', value='1', format='int')
45 # opObj11.addParameter(name='figpath', value=figpath, format='str')
46
47 # opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
48 # opObj11.addParameter(name='channelList', value='0', format='intList')
49 #
50 # opObj11.addParameter(name='id', value='2000', format='int')
51 # # opObj11.addParameter(name='colormap', value='0', format='bool')
52 # opObj11.addParameter(name='onlySNR', value='1', format='bool')
53 # opObj11.addParameter(name='DOP', value='0', format='bool')
54 # opObj11.addParameter(name='showSNR', value='1', format='bool')
55 # opObj11.addParameter(name='SNRthresh', value='0', format='int')
56 # opObj11.addParameter(name='SNRmin', value='-10', format='int')
57 # opObj11.addParameter(name='SNRmax', value='30', format='int')
58
59 # opObj11.addParameter(name='showSNR', value='1', format='int')
60 # # opObj11.addParameter(name='channelList', value='0', format='intlist')
61 # # opObj11.addParameter(name='xmin', value='0', format='float')
62 # opObj11.addParameter(name='xmin', value='0', format='float')
63 # opObj11.addParameter(name='xmax', value='24', format='float')
64
65 # opObj11.addParameter(name='zmin', value='-110', format='float')
66 # opObj11.addParameter(name='zmax', value='-70', format='float')
67 # opObj11.addParameter(name='save', value='0', format='int')
68 # # opObj11.addParameter(name='figpath', value='/tmp/', format='str')
69 #
70 opObj12 = procUnitConfObj2.addOperation(name='PublishData', optype='other')
71 opObj12.addParameter(name='zeromq', value=1, format='int')
72 # opObj12.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str')
73
74
75 # opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
76 # opObj13.addParameter(name='zeromq', value=1, format='int')
77 # opObj13.addParameter(name='server', value="juanca", format='str')
78
79 # opObj12.addParameter(name='delay', value=1, format='int')
80
81
82 # print "Escribiendo el archivo XML"
83 # controllerObj.writeXml(filename)
84 # print "Leyendo el archivo XML"
85 # controllerObj.readXml(filename)
86
87
88 # timeit.timeit('controllerObj.run()', number=2)
65
89
66 controllerObj.start()
90 controllerObj.start()
67
91
68 if __name__=='__main__':
69
70 dt = datetime(2017, 1, 12)
71
72 dates = [(dt+timedelta(x)).strftime('%Y/%m/%d') for x in range(20)]
73
92
74 p = multiprocessing.Pool(4)
93 if __name__ == '__main__':
75 p.map(main, dates)
94 parser = argparse.ArgumentParser(description='Set number of parallel processes')
95 parser.add_argument('--nProcess', default=1, type=int)
96 args = parser.parse_args()
97 multiSchain(fiber, nProcess=args.nProcess, startDate='2016/08/19', endDate='2016/08/19')
Show file before | Show file after | Hide comments
@@ -16,9 +16,9 if __name__ == '__main__':
16
16
17 proc1 = controllerObj.addProcUnit(name='ReceiverData')
17 proc1 = controllerObj.addProcUnit(name='ReceiverData')
18 proc1.addParameter(name='realtime', value='0', format='bool')
18 proc1.addParameter(name='realtime', value='0', format='bool')
19 proc1.addParameter(name='plottypes', value='rti,coh,phase,snr,dop', format='str')
19 proc1.addParameter(name='plottypes', value='rti,coh,phase', format='str')
20 proc1.addParameter(name='throttle', value='10', format='int')
20 proc1.addParameter(name='throttle', value='10', format='int')
21 proc1.addParameter(name='plot_server', value='tcp://10.10.10.82:7000', format='str')
21 # proc1.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str')
22 ## TODO Agregar direccion de server de publicacion a graficos como variable
22 ## TODO Agregar direccion de server de publicacion a graficos como variable
23
23
24 op1 = proc1.addOperation(name='PlotRTIData', optype='other')
24 op1 = proc1.addOperation(name='PlotRTIData', optype='other')
@@ -32,26 +32,26 if __name__ == '__main__':
32 op2.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
32 op2.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
33 op2.addParameter(name='colormap', value='jet', format='str')
33 op2.addParameter(name='colormap', value='jet', format='str')
34 op2.addParameter(name='show', value='0', format='bool')
34 op2.addParameter(name='show', value='0', format='bool')
35 # #
35 # # #
36 op6 = proc1.addOperation(name='PlotPHASEData', optype='other')
36 op6 = proc1.addOperation(name='PlotPHASEData', optype='other')
37 op6.addParameter(name='wintitle', value='Julia 150Km', format='str')
37 op6.addParameter(name='wintitle', value='Julia 150Km', format='str')
38 op6.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
38 op6.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
39 op6.addParameter(name='show', value='1', format='bool')
39 op6.addParameter(name='show', value='1', format='bool')
40 #
40 # #
41 # proc2 = controllerObj.addProcUnit(name='ReceiverData')
41 # # proc2 = controllerObj.addProcUnit(name='ReceiverData')
42 # proc2.addParameter(name='server', value='juanca', format='str')
42 # # proc2.addParameter(name='server', value='juanca', format='str')
43 # proc2.addParameter(name='plottypes', value='snr,dop', format='str')
43 # # proc2.addParameter(name='plottypes', value='snr,dop', format='str')
44 #
44 # #
45 op3 = proc1.addOperation(name='PlotSNRData', optype='other')
45 # op3 = proc1.addOperation(name='PlotSNRData', optype='other')
46 op3.addParameter(name='wintitle', value='Julia 150Km', format='str')
46 # op3.addParameter(name='wintitle', value='Julia 150Km', format='str')
47 op3.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
47 # op3.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
48 op3.addParameter(name='show', value='0', format='bool')
48 # op3.addParameter(name='show', value='0', format='bool')
49 #
49 # #
50 op4 = proc1.addOperation(name='PlotDOPData', optype='other')
50 # op4 = proc1.addOperation(name='PlotDOPData', optype='other')
51 op4.addParameter(name='wintitle', value='Julia 150Km', format='str')
51 # op4.addParameter(name='wintitle', value='Julia 150Km', format='str')
52 op4.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
52 # op4.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
53 op4.addParameter(name='show', value='0', format='bool')
53 # op4.addParameter(name='show', value='0', format='bool')
54 op4.addParameter(name='colormap', value='jet', format='str')
54 # op4.addParameter(name='colormap', value='jet', format='str')
55
55
56
56
57
57
Show file before | Show file after | Hide comments
@@ -1,1 +1,1
1 <Project description="HF_EXAMPLE" id="191" name="test01"><ReadUnit datatype="SpectraReader" id="1911" inputId="0" name="SpectraReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="SpectraReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/data/hysell_data20/pdata" /><Parameter format="date" id="191113" name="startDate" value="2015/09/26" /><Parameter format="date" id="191114" name="endDate" value="2015/09/26" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="cursor" value="2" /><Parameter format="int" id="191119" name="skip" value="720" /><Parameter format="int" id="191120" name="delay" value="10" /><Parameter format="int" id="191121" name="walk" value="1" /><Parameter format="int" id="191122" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="ParametersProc" id="1913" inputId="1911" name="ParametersProc"><Operation id="19131" name="run" priority="1" type="self" /><Operation id="19132" name="SpectralMoments" priority="2" type="other" /><Operation id="19133" name="PublishData" priority="3" type="other"><Parameter format="int" id="191331" name="zeromq" value="1" /><Parameter format="int" id="191332" name="delay" value="1" /></Operation></ProcUnit><ProcUnit datatype="Spectra" id="1912" inputId="1911" name="SpectraProc"><Operation id="19121" name="run" priority="1" type="self" /></ProcUnit></Project> No newline at end of file
1 <Project description="HF_EXAMPLE" id="191" name="test01"><ReadUnit datatype="SpectraReader" id="1911" inputId="0" name="SpectraReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="SpectraReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/data/sp1_f0" /><Parameter format="date" id="191113" name="startDate" value="2017/01/28" /><Parameter format="date" id="191114" name="endDate" value="2017/01/28" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="cursor" value="28" /><Parameter format="int" id="191119" name="skip" value="22" /><Parameter format="int" id="191120" name="walk" value="1" /><Parameter format="int" id="191121" name="verbose" value="1" /><Parameter format="int" id="191122" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="ParametersProc" id="1913" inputId="1911" name="ParametersProc"><Operation id="19131" name="run" priority="1" type="self" /><Operation id="19132" name="SpectralMoments" priority="2" type="other" /><Operation id="19133" name="PublishData" priority="3" type="other"><Parameter format="int" id="191331" name="zeromq" value="1" /><Parameter format="bool" id="191332" name="verbose" value="0" /><Parameter format="int" id="191333" name="delay" value="0" /></Operation></ProcUnit><ProcUnit datatype="Spectra" id="1912" inputId="1911" name="SpectraProc"><Operation id="19121" name="run" priority="1" type="self" /></ProcUnit></Project> No newline at end of file
Show file before | Show file after | Hide comments
@@ -44,5 +44,6 setup(name="schainpy",
44 "paramiko >= 2.1.2",
44 "paramiko >= 2.1.2",
45 "paho-mqtt >= 1.2",
45 "paho-mqtt >= 1.2",
46 "zmq",
46 "zmq",
47 "fuzzywuzzy"
47 ],
48 ],
48 ) No newline at end of file
49 )
General Comments 0
You need to be logged in to leave comments. Login now