| @@ -0,0 +1,82 | |||
|
|
1 | import argparse | |
|
|
2 | ||
|
|
3 | from schainpy.controller import Project, multiSchain | |
|
|
4 | ||
|
|
5 | desc = "HF_EXAMPLE" | |
|
|
6 | ||
|
|
7 | def fiber(cursor, skip, q, dt): | |
|
|
8 | ||
|
|
9 | controllerObj = Project() | |
|
|
10 | ||
|
|
11 | controllerObj.setup(id='191', name='test01', description=desc) | |
|
|
12 | ||
|
|
13 | readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader', | |
|
|
14 | path='/data/workspace/data/julia/', | |
|
|
15 | startDate=dt, | |
|
|
16 | endDate=dt, | |
|
|
17 | startTime="00:00:00", | |
|
|
18 | endTime="23:59:59", | |
|
|
19 | online=0, | |
|
|
20 | #set=1426485881, | |
|
|
21 | delay=10, | |
|
|
22 | walk=1, | |
|
|
23 | queue=q, | |
|
|
24 | cursor=cursor, | |
|
|
25 | skip=skip, | |
|
|
26 | #timezone=-5*3600 | |
|
|
27 | ) | |
|
|
28 | ||
|
|
29 | # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock') | |
|
|
30 | # | |
|
|
31 | procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId()) | |
|
|
32 | # opObj11 = procUnitConfObj2.addParameter(name='pairsList', value='(0,1)', format='pairslist') | |
|
|
33 | # | |
|
|
34 | # procUnitConfObj2 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId()) | |
|
|
35 | ||
|
|
36 | # opObj11 = procUnitConfObj2.addOperation(name='SpectralMoments', optype='other') | |
|
|
37 | ||
|
|
38 | # | |
|
|
39 | # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other') | |
|
|
40 | # opObj11.addParameter(name='id', value='1000', format='int') | |
|
|
41 | # opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str') | |
|
|
42 | # opObj11.addParameter(name='channelList', value='0', format='intlist') | |
|
|
43 | # opObj11.addParameter(name='zmin', value='-120', format='float') | |
|
|
44 | # opObj11.addParameter(name='zmax', value='-70', format='float') | |
|
|
45 | # opObj11.addParameter(name='save', value='1', format='int') | |
|
|
46 | # opObj11.addParameter(name='figpath', value=figpath, format='str') | |
|
|
47 | ||
|
|
48 | # opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other') | |
|
|
49 | # opObj11.addParameter(name='id', value='2000', format='int') | |
|
|
50 | # opObj11.addParameter(name='wintitle', value='HF_Jicamarca', format='str') | |
|
|
51 | # opObj11.addParameter(name='showprofile', value='0', format='int') | |
|
|
52 | # opObj11.addParameter(name='channelList', value='0', format='intlist') | |
|
|
53 | # # opObj11.addParameter(name='xmin', value='0', format='float') | |
|
|
54 | # opObj11.addParameter(name='xmin', value='0', format='float') | |
|
|
55 | # opObj11.addParameter(name='xmax', value='24', format='float') | |
|
|
56 | # | |
|
|
57 | # opObj11.addParameter(name='zmin', value='-110', format='float') | |
|
|
58 | # opObj11.addParameter(name='zmax', value='-70', format='float') | |
|
|
59 | # opObj11.addParameter(name='save', value='0', format='int') | |
|
|
60 | # opObj11.addParameter(name='figpath', value='/tmp/', format='str') | |
|
|
61 | ||
|
|
62 | opObj12 = procUnitConfObj2.addOperation(name='PublishData', optype='other') | |
|
|
63 | opObj12.addParameter(name='zeromq', value=1, format='int') | |
|
|
64 | ||
|
|
65 | # print "Escribiendo el archivo XML" | |
|
|
66 | # controllerObj.writeXml(filename) | |
|
|
67 | # print "Leyendo el archivo XML" | |
|
|
68 | # controllerObj.readXml(filename) | |
|
|
69 | ||
|
|
70 | controllerObj.createObjects() | |
|
|
71 | controllerObj.connectObjects() | |
|
|
72 | ||
|
|
73 | # timeit.timeit('controllerObj.run()', number=2) | |
|
|
74 | ||
|
|
75 | controllerObj.run() | |
|
|
76 | ||
|
|
77 | ||
|
|
78 | if __name__ == '__main__': | |
|
|
79 | parser = argparse.ArgumentParser(description='Set number of parallel processes') | |
|
|
80 | parser.add_argument('--nProcess', default=2, type=int) | |
|
|
81 | args = parser.parse_args() | |
|
|
82 | multiSchain(fiber, nProcess=args.nProcess, startDate='2016/08/19', endDate='2016/08/20') | |
| @@ -7,6 +7,8 import sys | |||
|
|
7 | 7 | import ast |
|
|
8 | 8 | import datetime |
|
|
9 | 9 | import traceback |
|
|
10 | from multiprocessing import Process, Queue, cpu_count | |
|
|
11 | ||
|
|
10 | 12 | import schainpy |
|
|
11 | 13 | import schainpy.admin |
|
|
12 | 14 | |
| @@ -23,6 +25,51 def prettify(elem): | |||
|
|
23 | 25 | reparsed = minidom.parseString(rough_string) |
|
|
24 | 26 | return reparsed.toprettyxml(indent=" ") |
|
|
25 | 27 | |
|
|
28 | def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None): | |
|
|
29 | skip = 0 | |
|
|
30 | cursor = 0 | |
|
|
31 | nFiles = None | |
|
|
32 | processes = [] | |
|
|
33 | ||
|
|
34 | dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d') | |
|
|
35 | dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d') | |
|
|
36 | days = (dt2 - dt1).days | |
|
|
37 | print days | |
|
|
38 | for day in range(days+1): | |
|
|
39 | skip = 0 | |
|
|
40 | cursor = 0 | |
|
|
41 | q = Queue() | |
|
|
42 | processes = [] | |
|
|
43 | dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d') | |
|
|
44 | firstProcess = Process(target=child, args=(cursor, skip, q, dt)) | |
|
|
45 | firstProcess.start() | |
|
|
46 | nFiles = q.get() | |
|
|
47 | firstProcess.terminate() | |
|
|
48 | skip = int(math.ceil(nFiles/nProcess)) | |
|
|
49 | try: | |
|
|
50 | while True: | |
|
|
51 | processes.append(Process(target=child, args=(cursor, skip, q, dt))) | |
|
|
52 | processes[cursor].start() | |
|
|
53 | if nFiles < cursor*skip: | |
|
|
54 | break | |
|
|
55 | cursor += 1 | |
|
|
56 | except KeyboardInterrupt: | |
|
|
57 | for process in processes: | |
|
|
58 | process.terminate() | |
|
|
59 | process.join() | |
|
|
60 | for process in processes: | |
|
|
61 | process.join() | |
|
|
62 | #process.terminate() | |
|
|
63 | sleep(3) | |
|
|
64 | ||
|
|
65 | try: | |
|
|
66 | while True: | |
|
|
67 | pass | |
|
|
68 | except KeyboardInterrupt: | |
|
|
69 | for process in processes: | |
|
|
70 | process.terminate() | |
|
|
71 | process.join() | |
|
|
72 | ||
|
|
26 | 73 | class ParameterConf(): |
|
|
27 | 74 | |
|
|
28 | 75 | id = None |
| @@ -51,6 +98,9 class ParameterConf(): | |||
|
|
51 | 98 | |
|
|
52 | 99 | return self.__formated_value |
|
|
53 | 100 | |
|
|
101 | if format == 'obj': | |
|
|
102 | return value | |
|
|
103 | ||
|
|
54 | 104 | if format == 'str': |
|
|
55 | 105 | self.__formated_value = str(value) |
|
|
56 | 106 | return self.__formated_value |
| @@ -171,6 +221,9 class ParameterConf(): | |||
|
|
171 | 221 | |
|
|
172 | 222 | self.id = str(id) |
|
|
173 | 223 | self.name = name |
|
|
224 | if format == 'obj': | |
|
|
225 | self.value = value | |
|
|
226 | else: | |
|
|
174 | 227 | self.value = str(value) |
|
|
175 | 228 | self.format = str.lower(format) |
|
|
176 | 229 | |
| @@ -698,7 +751,7 class ReadUnitConf(ProcUnitConf): | |||
|
|
698 | 751 | |
|
|
699 | 752 | return self.ELEMENTNAME |
|
|
700 | 753 | |
|
|
701 | def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, **kwargs): | |
|
|
754 | def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, queue=None, **kwargs): | |
|
|
702 | 755 | |
|
|
703 | 756 | #Compatible with old signal chain version |
|
|
704 | 757 | if datatype==None and name==None: |
| @@ -725,7 +778,7 class ReadUnitConf(ProcUnitConf): | |||
|
|
725 | 778 | |
|
|
726 | 779 | self.inputId = '0' |
|
|
727 | 780 | self.parentId = parentId |
|
|
728 | ||
|
|
781 | self.queue = queue | |
|
|
729 | 782 | self.addRunOperation(**kwargs) |
|
|
730 | 783 | |
|
|
731 | 784 | def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs): |
| @@ -84,7 +84,8 class PlotData(Operation, Process): | |||
|
|
84 | 84 | print 'plotting...{}'.format(self.CODE) |
|
|
85 | 85 | |
|
|
86 | 86 | self.plot() |
|
|
87 |
self.figure.suptitle('{} {}'.format(self.title, self.CODE.upper() |
|
|
|
87 | self.figure.suptitle('{} {} - Date:{}'.format(self.title, self.CODE.upper(), | |
|
|
88 | datetime.datetime.utcfromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S'))) | |
|
|
88 | 89 | |
|
|
89 | 90 | if self.save: |
|
|
90 | 91 | figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE, |
| @@ -234,17 +235,17 class PlotSpectraData(PlotData): | |||
|
|
234 | 235 | ax.ax_profile.set_ylim(self.ymin, self.ymax) |
|
|
235 | 236 | ax.ax_profile.set_xlabel('dB') |
|
|
236 | 237 | ax.ax_profile.grid(b=True, axis='x') |
|
|
238 | ax.plot_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y, | |
|
|
239 | color="k", linestyle="dashed", lw=2)[0] | |
|
|
237 | 240 | [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()] |
|
|
238 | noise = 10*numpy.log10(self.data['rti'][self.max_time][n]/self.dataOut.normFactor) | |
|
|
239 | ax.ax_profile.vlines(noise, self.ymin, self.ymax, colors="k", linestyle="dashed", lw=2) | |
|
|
240 | 241 | else: |
|
|
241 | 242 | ax.plot.set_array(z[n].T.ravel()) |
|
|
242 | ax.set_title('{} {}'.format(self.titles[n], | |
|
|
243 | datetime.datetime.utcfromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')), | |
|
|
244 | size=8) | |
|
|
245 | 243 | if self.showprofile: |
|
|
246 | 244 | ax.plot_profile.set_data(self.data['rti'][self.max_time][n], y) |
|
|
245 | ax.plot_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y) | |
|
|
247 | 246 | |
|
|
247 | ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]), | |
|
|
248 | size=8) | |
|
|
248 | 249 | |
|
|
249 | 250 | class PlotRTIData(PlotData): |
|
|
250 | 251 | |
| @@ -608,6 +608,9 class JRODataReader(JRODataIO): | |||
|
|
608 | 608 | set=None, |
|
|
609 | 609 | expLabel='', |
|
|
610 | 610 | ext='.r', |
|
|
611 | queue=None, | |
|
|
612 | cursor=None, | |
|
|
613 | skip=None, | |
|
|
611 | 614 | walk=True): |
|
|
612 | 615 | |
|
|
613 | 616 | self.filenameList = [] |
| @@ -635,7 +638,21 class JRODataReader(JRODataIO): | |||
|
|
635 | 638 | fileList = glob.glob1(thisPath, "*%s" %ext) |
|
|
636 | 639 | fileList.sort() |
|
|
637 | 640 | |
|
|
638 |
|
|
|
|
641 | skippedFileList = [] | |
|
|
642 | ||
|
|
643 | if cursor is not None and skip is not None: | |
|
|
644 | # if cursor*skip > len(fileList): | |
|
|
645 | if skip == 0: | |
|
|
646 | if queue is not None: | |
|
|
647 | queue.put(len(fileList)) | |
|
|
648 | skippedFileList = [] | |
|
|
649 | else: | |
|
|
650 | skippedFileList = fileList[cursor*skip: cursor*skip + skip] | |
|
|
651 | ||
|
|
652 | else: | |
|
|
653 | skippedFileList = fileList | |
|
|
654 | ||
|
|
655 | for file in skippedFileList: | |
|
|
639 | 656 | |
|
|
640 | 657 | filename = os.path.join(thisPath,file) |
|
|
641 | 658 | |
| @@ -1246,7 +1263,10 class JRODataReader(JRODataIO): | |||
|
|
1246 | 1263 | nTxs = 1, |
|
|
1247 | 1264 | realtime=False, |
|
|
1248 | 1265 | blocksize=None, |
|
|
1249 |
blocktime=None |
|
|
|
1266 | blocktime=None, | |
|
|
1267 | queue=None, | |
|
|
1268 | skip=None, | |
|
|
1269 | cursor=None): | |
|
|
1250 | 1270 | |
|
|
1251 | 1271 | if path == None: |
|
|
1252 | 1272 | raise ValueError, "[Reading] The path is not valid" |
| @@ -1282,7 +1302,8 class JRODataReader(JRODataIO): | |||
|
|
1282 | 1302 | pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate, |
|
|
1283 | 1303 | startTime=startTime, endTime=endTime, |
|
|
1284 | 1304 | set=set, expLabel=expLabel, ext=ext, |
|
|
1285 |
walk=walk |
|
|
|
1305 | walk=walk, cursor=cursor, | |
|
|
1306 | skip=skip, queue=queue) | |
|
|
1286 | 1307 | |
|
|
1287 | 1308 | if not(pathList): |
|
|
1288 | 1309 | # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path, |
| @@ -1720,4 +1741,3 class JRODataWriter(JRODataIO): | |||
|
|
1720 | 1741 | self.isConfig = True |
|
|
1721 | 1742 | |
|
|
1722 | 1743 | self.putData() |
|
|
1723 | ||
| @@ -279,7 +279,7 class ReceiverData(ProcessingUnit, Process): | |||
|
|
279 | 279 | self.data['times'] = [] |
|
|
280 | 280 | for plottype in self.plottypes: |
|
|
281 | 281 | self.data[plottype] = {} |
|
|
282 | ||
|
|
282 | self.data['noise'] = {} | |
|
|
283 | 283 | self.isConfig = True |
|
|
284 | 284 | |
|
|
285 | 285 | def event_monitor(self, monitor): |
| @@ -325,8 +325,8 class ReceiverData(ProcessingUnit, Process): | |||
|
|
325 | 325 | |
|
|
326 | 326 | if plottype == 'spc': |
|
|
327 | 327 | z = self.dataOut.data_spc/self.dataOut.normFactor |
|
|
328 |
|
|
|
|
329 | self.data[plottype] = zdB | |
|
|
328 | self.data[plottype] = 10*numpy.log10(z) | |
|
|
329 | self.data['noise'][t] = 10*numpy.log10(self.dataOut.getNoise()/self.dataOut.normFactor) | |
|
|
330 | 330 | if plottype == 'rti': |
|
|
331 | 331 | self.data[plottype][t] = self.dataOut.getPower() |
|
|
332 | 332 | if plottype == 'snr': |
General Comments 0
You need to be logged in to leave comments.
Login now