##// END OF EJS Templates
schain
RSS
funcionando todo
José Chávez -
r898:5b95b19a52cf
parent child
Show More
Show file before | Show file after | Hide comments
@@ -7,6 +7,7 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 import math
10 11 from multiprocessing import Process, Queue, cpu_count
11 12
12 13 import schainpy
@@ -25,7 +26,7 def prettify(elem):
25 26 reparsed = minidom.parseString(rough_string)
26 27 return reparsed.toprettyxml(indent=" ")
27 28
28 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None):
29 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, receiver=None):
29 30 skip = 0
30 31 cursor = 0
31 32 nFiles = None
@@ -43,10 +44,7 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None):
43 44 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
44 45 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
45 46 firstProcess.start()
46 print 'a'
47 47 nFiles = q.get()
48
49 print nFiles
50 48 firstProcess.terminate()
51 49 skip = int(math.ceil(nFiles/nProcess))
52 50 try:
@@ -241,7 +239,7 class ParameterConf():
241 239 self.format = format
242 240
243 241 def makeXml(self, opElement):
244
242 if self.name not in ('queue',):
245 243 parmElement = SubElement(opElement, self.ELEMENTNAME)
246 244 parmElement.set('id', str(self.id))
247 245 parmElement.set('name', self.name)
Show file before | Show file after | Hide comments
@@ -50,7 +50,7 class PlotData(Operation, Process):
50 50 self.xrange = kwargs.get('xrange', 24)
51 51 self.ymin = kwargs.get('ymin', None)
52 52 self.ymax = kwargs.get('ymax', None)
53
53 self.throttle_value = 1
54 54 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
55 55
56 56 if x_buffer.shape[0] < 2:
@@ -71,12 +71,13 class PlotData(Operation, Process):
71 71
72 72 def decimate(self):
73 73
74 dx = int(len(self.x)/self.__MAXNUMX) + 1
74 # dx = int(len(self.x)/self.__MAXNUMX) + 1
75 75 dy = int(len(self.y)/self.__MAXNUMY) + 1
76 76
77 x = self.x[::dx]
77 # x = self.x[::dx]
78 x = self.x
78 79 y = self.y[::dy]
79 z = self.z[::, ::dx, ::dy]
80 z = self.z[::, ::, ::dy]
80 81
81 82 return x, y, z
82 83
@@ -90,7 +91,7 class PlotData(Operation, Process):
90 91
91 92 if self.save:
92 93 figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,
93 datetime.datetime.utcfromtimestamp(self.times[-1]).strftime('%y%m%d_%H%M%S')))
94 datetime.datetime.utcfromtimestamp(self.times[0]).strftime('%y%m%d_%H%M%S')))
94 95 print 'Saving figure: {}'.format(figname)
95 96 self.figure.savefig(figname)
96 97
@@ -117,23 +118,22 class PlotData(Operation, Process):
117 118 self.dataOut = self.data['dataOut']
118 119 self.times = self.data['times']
119 120 self.times.sort()
121 self.throttle_value = self.data['throttle']
120 122 self.min_time = self.times[0]
121 123 self.max_time = self.times[-1]
122 124
123 125 if self.isConfig is False:
124 126 self.setup()
125 127 self.isConfig = True
126
127 128 self.__plot()
128 129
129 if 'ENDED' in self.data:
130 # self.setup()
130 if self.data['ENDED'] is True:
131 131 # self.__plot()
132 pass
132 self.isConfig = False
133 133
134 134 except zmq.Again as e:
135 135 print 'Waiting for data...'
136 plt.pause(5)
136 plt.pause(self.throttle_value)
137 137 #time.sleep(3)
138 138
139 139 def close(self):
@@ -254,7 +254,6 class PlotRTIData(PlotData):
254 254 colormap = 'jro'
255 255
256 256 def setup(self):
257
258 257 self.ncols = 1
259 258 self.nrows = self.dataOut.nChannels
260 259 self.width = 10
@@ -268,12 +267,12 class PlotRTIData(PlotData):
268 267 facecolor='w')
269 268 else:
270 269 self.figure.clf()
270 self.axes = []
271 271
272 272 for n in range(self.nrows):
273 273 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
274 274 ax.firsttime = True
275 275 self.axes.append(ax)
276
277 276 self.figure.subplots_adjust(hspace=0.5)
278 277 self.figure.show()
279 278
@@ -287,16 +286,16 class PlotRTIData(PlotData):
287 286 self.z.append([self.data[self.CODE][t][ch] for t in self.times])
288 287
289 288 self.z = np.array(self.z)
290
291 289 for n, ax in enumerate(self.axes):
292 290
293 291 x, y, z = self.fill_gaps(*self.decimate())
294
292 xmin = self.min_time
293 xmax = xmin+self.xrange*60*60
295 294 if ax.firsttime:
296 295 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
297 296 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
298 297 self.zmin = self.zmin if self.zmin else np.nanmin(self.z)
299 zmax = self.zmax if self.zmax else np.nanmax(self.z)
298 self.zmax = self.zmax if self.zmax else np.nanmax(self.z)
300 299 plot = ax.pcolormesh(x, y, z[n].T,
301 300 vmin=self.zmin,
302 301 vmax=self.zmax,
@@ -311,24 +310,21 class PlotRTIData(PlotData):
311 310 ax.xaxis.set_major_formatter(FuncFormatter(func))
312 311 ax.xaxis.set_major_locator(LinearLocator(6))
313 312
314 ax.yaxis.set_major_locator(LinearLocator(4))
313 # ax.yaxis.set_major_locator(LinearLocator(4))
315 314
316 315 ax.set_ylabel(self.ylabel)
317 316
318 if self.xmin is None:
319 print 'is none'
320 xmin = self.min_time
321 else:
322
323 xmin = (datetime.datetime.combine(self.dataOut.datatime.date(),
324 datetime.time(self.xmin, 0, 0))-d1970).total_seconds()
325
326 xmax = xmin+self.xrange*60*60
317 # if self.xmin is None:
318 # xmin = self.min_time
319 # else:
320 # xmin = (datetime.datetime.combine(self.dataOut.datatime.date(),
321 # datetime.time(self.xmin, 0, 0))-d1970).total_seconds()
327 322
328 323 ax.set_xlim(xmin, xmax)
329 324 ax.firsttime = False
330 325 else:
331 326 ax.collections.remove(ax.collections[0])
327 ax.set_xlim(xmin, xmax)
332 328 plot = ax.pcolormesh(x, y, z[n].T,
333 329 vmin=self.zmin,
334 330 vmax=self.zmax,
@@ -369,8 +365,11 class PlotCOHData(PlotRTIData):
369 365
370 366 class PlotSNRData(PlotRTIData):
371 367
372 CODE = 'coh'
368 CODE = 'snr'
373 369
370 class PlotDOPData(PlotRTIData):
371 CODE = 'dop'
372 colormap = 'jet'
374 373
375 374 class PlotPHASEData(PlotCOHData):
376 375
Show file before | Show file after | Hide comments
@@ -156,6 +156,7 class SpectralMoments(Operation):
156 156
157 157 dataOut.data_param = data_param[:,1:,:]
158 158 dataOut.data_SNR = data_param[:,0]
159 dataOut.data_DOP = data_param[:,1]
159 160 return
160 161
161 162 def __calculateMoments(self, oldspec, oldfreq, n0, nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None):
@@ -2743,4 +2744,3 class SMOperations():
2743 2744 # error[indInvalid1] = 13
2744 2745 #
2745 2746 # return heights, error
2746 No newline at end of file
Show file before | Show file after | Hide comments
@@ -18,7 +18,6 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
18 18
19 19 MAXNUMX = 100
20 20 MAXNUMY = 100
21 throttle_value = 5
22 21
23 22 class PrettyFloat(float):
24 23 def __repr__(self):
@@ -49,6 +48,7 class throttle(object):
49 48 self.throttle_period = datetime.timedelta(
50 49 seconds=seconds, minutes=minutes, hours=hours
51 50 )
51
52 52 self.time_of_last_call = datetime.datetime.min
53 53
54 54 def __call__(self, fn):
@@ -91,7 +91,6 class PublishData(Operation):
91 91 port=self.port,
92 92 keepalive=60*10,
93 93 bind_address='')
94 print "connected"
95 94 self.client.loop_start()
96 95 # self.client.publish(
97 96 # self.topic + 'SETUP',
@@ -116,7 +115,6 class PublishData(Operation):
116 115 self.client = None
117 116 setup = []
118 117 if mqtt is 1:
119 print 'mqqt es 1'
120 118 self.client = mqtt.Client(
121 119 client_id=self.clientId + self.topic + 'SCHAIN',
122 120 clean_session=True)
@@ -145,7 +143,6 class PublishData(Operation):
145 143
146 144 self.zmq_socket.connect(address)
147 145 time.sleep(1)
148 print 'zeromq configured'
149 146
150 147
151 148 def publish_data(self):
@@ -252,6 +249,8 class PublishData(Operation):
252 249
253 250 class ReceiverData(ProcessingUnit, Process):
254 251
252 throttle_value = 5
253
255 254 def __init__(self, **kwargs):
256 255
257 256 ProcessingUnit.__init__(self, **kwargs)
@@ -269,8 +268,8 class ReceiverData(ProcessingUnit, Process):
269 268 self.address = address
270 269 self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
271 270 self.realtime = kwargs.get('realtime', False)
272 global throttle_value
273 throttle_value = kwargs.get('throttle', 10)
271 self.throttle_value = kwargs.get('throttle', 10)
272 self.sendData = self.initThrottle(self.throttle_value)
274 273 self.setup()
275 274
276 275 def setup(self):
@@ -280,6 +279,8 class ReceiverData(ProcessingUnit, Process):
280 279 for plottype in self.plottypes:
281 280 self.data[plottype] = {}
282 281 self.data['noise'] = {}
282 self.data['throttle'] = self.throttle_value
283 self.data['ENDED'] = False
283 284 self.isConfig = True
284 285
285 286 def event_monitor(self, monitor):
@@ -305,11 +306,14 class ReceiverData(ProcessingUnit, Process):
305 306 if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
306 307 break
307 308 monitor.close()
308 print("event monitor thread done!")
309
310 def initThrottle(self, throttle_value):
309 311
310 312 @throttle(seconds=throttle_value)
311 def sendData(self, data):
312 self.send(data)
313 def sendDataThrottled(fn_sender, data):
314 fn_sender(data)
315
316 return sendDataThrottled
313 317
314 318 def send(self, data):
315 319 print '[sending] data=%s size=%s' % (data.keys(), len(data['times']))
@@ -355,8 +359,8 class ReceiverData(ProcessingUnit, Process):
355 359
356 360 while True:
357 361 self.dataOut = self.receiver.recv_pyobj()
358 print '[Receiving] {} - {}'.format(self.dataOut.type,
359 self.dataOut.datatime.ctime())
362 # print '[Receiving] {} - {}'.format(self.dataOut.type,
363 # self.dataOut.datatime.ctime())
360 364
361 365 self.update()
362 366
@@ -372,7 +376,7 class ReceiverData(ProcessingUnit, Process):
372 376 if self.realtime:
373 377 self.send(self.data)
374 378 else:
375 self.sendData(self.data)
379 self.sendData(self.send, self.data)
376 380 self.started = True
377 381
378 382 return
Show file before | Show file after | Hide comments
@@ -11,7 +11,7 def fiber(cursor, skip, q, dt):
11 11 controllerObj.setup(id='191', name='test01', description=desc)
12 12
13 13 readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
14 path='/home/nanosat/data/julia',
14 path='/home/nanosat/data/hysell_data20/pdata',
15 15 startDate=dt,
16 16 endDate=dt,
17 17 startTime="00:00:00",
@@ -31,9 +31,9 def fiber(cursor, skip, q, dt):
31 31 procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
32 32 # opObj11 = procUnitConfObj2.addParameter(name='pairsList', value='(0,1)', format='pairslist')
33 33 #
34 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
34 procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
35 35
36 # opObj11 = procUnitConfObj2.addOperation(name='SpectralMoments', optype='other')
36 opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
37 37
38 38 #
39 39 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
@@ -45,14 +45,14 def fiber(cursor, skip, q, dt):
45 45 # opObj11.addParameter(name='save', value='1', format='int')
46 46 # opObj11.addParameter(name='figpath', value=figpath, format='str')
47 47
48 opObj11 = procUnitConfObj2.addOperation(name='RTIPlot', optype='other')
49 opObj11.addParameter(name='id', value='2000', format='int')
50 opObj11.addParameter(name='wintitzmaxle', value='HF_Jicamarca', format='str')
51 opObj11.addParameter(name='showprofile', value='0', format='int')
52 # opObj11.addParameter(name='channelList', value='0', format='intlist')
48 # opObj11 = procUnitConfObj2.addOperation(name='RTIPlot', optype='other')
49 # opObj11.addParameter(name='id', value='2000', format='int')
50 # opObj11.addParameter(name='wintitzmaxle', value='HF_Jicamarca', format='str')
51 # opObj11.addParameter(name='showprofile', value='0', format='int')
52 # # opObj11.addParameter(name='channelList', value='0', format='intlist')
53 # # opObj11.addParameter(name='xmin', value='0', format='float')
53 54 # opObj11.addParameter(name='xmin', value='0', format='float')
54 opObj11.addParameter(name='xmin', value='0', format='float')
55 opObj11.addParameter(name='xmax', value='24', format='float')
55 # opObj11.addParameter(name='xmax', value='24', format='float')
56 56
57 57 # opObj11.addParameter(name='zmin', value='-110', format='float')
58 58 # opObj11.addParameter(name='zmax', value='-70', format='float')
@@ -62,21 +62,26 def fiber(cursor, skip, q, dt):
62 62 opObj12 = procUnitConfObj2.addOperation(name='PublishData', optype='other')
63 63 opObj12.addParameter(name='zeromq', value=1, format='int')
64 64
65 opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
66 opObj13.addParameter(name='zeromq', value=1, format='int')
67 opObj13.addParameter(name='server', value="juanca", format='str')
68
69 # opObj12.addParameter(name='delay', value=1, format='int')
70
71
65 72 # print "Escribiendo el archivo XML"
66 73 # controllerObj.writeXml(filename)
67 74 # print "Leyendo el archivo XML"
68 75 # controllerObj.readXml(filename)
69 76
70 controllerObj.createObjects()
71 controllerObj.connectObjects()
72 77
73 78 # timeit.timeit('controllerObj.run()', number=2)
74 79
75 controllerObj.run()
80 controllerObj.start()
76 81
77 82
78 83 if __name__ == '__main__':
79 84 parser = argparse.ArgumentParser(description='Set number of parallel processes')
80 parser.add_argument('--nProcess', default=1, type=int)
85 parser.add_argument('--nProcess', default=16, type=int)
81 86 args = parser.parse_args()
82 multiSchain(fiber, nProcess=args.nProcess, startDate='2016/08/19', endDate='2016/08/20')
87 multiSchain(fiber, nProcess=args.nProcess, startDate='2015/09/26', endDate='2015/09/26')
Show file before | Show file after | Hide comments
@@ -15,27 +15,35 if __name__ == '__main__':
15 15 controllerObj.setup(id='191', name='test01', description=desc)
16 16
17 17 proc1 = controllerObj.addProcUnit(name='ReceiverData')
18 # proc1.addParameter(name='server', value='tcp://10.10.10.87:3000', format='str')
19 proc1.addParameter(name='realtime', value='1', format='bool')
20 proc1.addParameter(name='plottypes', value='spc', format='str')
21
22 # op1 = proc1.addOperation(name='PlotRTIData', optype='other')
23 # op1.addParameter(name='wintitle', value='Julia 150Km', format='str')
24 #
25 op2 = proc1.addOperation(name='PlotSpectraData', optype='other')
18 proc1.addParameter(name='realtime', value='0', format='bool')
19 proc1.addParameter(name='plottypes', value='rti,coh,phase', format='str')
20 proc1.addParameter(name='throttle', value='10', format='int')
21
22 op1 = proc1.addOperation(name='PlotRTIData', optype='other')
23 op1.addParameter(name='wintitle', value='Julia 150Km', format='str')
24 op1.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
25 op1.addParameter(name='colormap', value='jet', format='str')
26
27 op2 = proc1.addOperation(name='PlotCOHData', optype='other')
26 28 op2.addParameter(name='wintitle', value='Julia 150Km', format='str')
27 # op2.addParameter(name='xaxis', value='velocity', format='str')
28 # op2.addParameter(name='showprofile', value='1', format='bool')
29 #op2.addParameter(name='xmin', value='-0.1', format='float')
30 #op2.addParameter(name='xmax', value='0.1', format='float')
31
32 # op1 = proc1.addOperation(name='PlotPHASEData', optype='other')
33 # op1.addParameter(name='wintitle', value='Julia 150Km', format='str')
34
35 # proc1 = controllerObj.addProcUnit(name='ReceiverData')
36 # proc1.addParameter(name='server', value='pipe2', format='str')
37 # proc1.addParameter(name='mode', value='buffer', format='str')
38 # proc1.addParameter(name='plottypes', value='snr', format='str')
29 op2.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
30
31 op6 = proc1.addOperation(name='PlotPHASEData', optype='other')
32 op6.addParameter(name='wintitle', value='Julia 150Km', format='str')
33 op6.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
34
35 proc2 = controllerObj.addProcUnit(name='ReceiverData')
36 proc2.addParameter(name='server', value='juanca', format='str')
37 proc2.addParameter(name='plottypes', value='snr,dop', format='str')
38
39 op3 = proc2.addOperation(name='PlotSNRData', optype='other')
40 op3.addParameter(name='wintitle', value='Julia 150Km', format='str')
41 op3.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
42
43 op4 = proc2.addOperation(name='PlotDOPData', optype='other')
44 op4.addParameter(name='wintitle', value='Julia 150Km', format='str')
45 op4.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
46
39 47
40 48
41 49 controllerObj.start()
Show file before | Show file after | Hide comments
@@ -1,1 +1,1
1 <Project description="" id="191" name="test01"><ReadUnit datatype="Spectra" id="1911" inputId="0" name="SpectraReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="Spectra" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/data/zeus" /><Parameter format="date" id="191113" name="startDate" value="2017/01/30" /><Parameter format="date" id="191114" name="endDate" value="2017/01/30" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="obj" id="191117" name="queue" No newline at end of file
1 <Project description="HF_EXAMPLE" id="191" name="test01"><ReadUnit datatype="SpectraReader" id="1911" inputId="0" name="SpectraReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="SpectraReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/data/hysell_data20/pdata" /><Parameter format="date" id="191113" name="startDate" value="2015/09/26" /><Parameter format="date" id="191114" name="endDate" value="2015/09/26" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="cursor" value="17" /><Parameter format="int" id="191119" name="skip" value="45" /><Parameter format="int" id="191120" name="delay" value="10" /><Parameter format="int" id="191121" name="walk" value="1" /><Parameter format="int" id="191122" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="ParametersProc" id="1913" inputId="1911" name="ParametersProc"><Operation id="19131" name="run" priority="1" type="self" /><Operation id="19132" name="SpectralMoments" priority="2" type="other" /><Operation id="19133" name="PublishData" priority="3" type="other"><Parameter format="int" id="191331" name="zeromq" value="1" /><Parameter format="str" id="191332" name="server" value="juanca" /></Operation></ProcUnit><ProcUnit datatype="Spectra" id="1912" inputId="1911" name="SpectraProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="PublishData" priority="2" type="other"><Parameter format="int" id="191221" name="zeromq" value="1" /></Operation></ProcUnit></Project> No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now