| @@ -7,10 +7,6 import sys | |||||
| 7 | import ast |
|
7 | import ast | |
| 8 | import datetime |
|
8 | import datetime | |
| 9 | import traceback |
|
9 | import traceback | |
| 10 | import math |
|
|||
| 11 | import time |
|
|||
| 12 | from multiprocessing import Process, Queue, cpu_count |
|
|||
| 13 |
|
||||
| 14 | import schainpy |
|
10 | import schainpy | |
| 15 | import schainpy.admin |
|
11 | import schainpy.admin | |
| 16 |
|
12 | |||
| @@ -27,48 +23,6 def prettify(elem): | |||||
| 27 | reparsed = minidom.parseString(rough_string) |
|
23 | reparsed = minidom.parseString(rough_string) | |
| 28 | return reparsed.toprettyxml(indent=" ") |
|
24 | return reparsed.toprettyxml(indent=" ") | |
| 29 |
|
25 | |||
| 30 | def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False): |
|
|||
| 31 | skip = 0 |
|
|||
| 32 | cursor = 0 |
|
|||
| 33 | nFiles = None |
|
|||
| 34 | processes = [] |
|
|||
| 35 | dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d') |
|
|||
| 36 | dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d') |
|
|||
| 37 | days = (dt2 - dt1).days |
|
|||
| 38 |
|
||||
| 39 | for day in range(days+1): |
|
|||
| 40 | skip = 0 |
|
|||
| 41 | cursor = 0 |
|
|||
| 42 | q = Queue() |
|
|||
| 43 | processes = [] |
|
|||
| 44 | dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d') |
|
|||
| 45 | firstProcess = Process(target=child, args=(cursor, skip, q, dt)) |
|
|||
| 46 | firstProcess.start() |
|
|||
| 47 | if by_day: |
|
|||
| 48 | continue |
|
|||
| 49 | nFiles = q.get() |
|
|||
| 50 | firstProcess.terminate() |
|
|||
| 51 | skip = int(math.ceil(nFiles/nProcess)) |
|
|||
| 52 | while True: |
|
|||
| 53 | processes.append(Process(target=child, args=(cursor, skip, q, dt))) |
|
|||
| 54 | processes[cursor].start() |
|
|||
| 55 | if nFiles < cursor*skip: |
|
|||
| 56 | break |
|
|||
| 57 | cursor += 1 |
|
|||
| 58 |
|
||||
| 59 | def beforeExit(exctype, value, trace): |
|
|||
| 60 | for process in processes: |
|
|||
| 61 | process.terminate() |
|
|||
| 62 | process.join() |
|
|||
| 63 | print traceback.print_tb(trace) |
|
|||
| 64 |
|
||||
| 65 | sys.excepthook = beforeExit |
|
|||
| 66 |
|
||||
| 67 | for process in processes: |
|
|||
| 68 | process.join() |
|
|||
| 69 | process.terminate() |
|
|||
| 70 | time.sleep(3) |
|
|||
| 71 |
|
||||
| 72 | class ParameterConf(): |
|
26 | class ParameterConf(): | |
| 73 |
|
27 | |||
| 74 | id = None |
|
28 | id = None | |
| @@ -97,9 +51,6 class ParameterConf(): | |||||
| 97 |
|
51 | |||
| 98 | return self.__formated_value |
|
52 | return self.__formated_value | |
| 99 |
|
53 | |||
| 100 | if format == 'obj': |
|
|||
| 101 | return value |
|
|||
| 102 |
|
||||
| 103 | if format == 'str': |
|
54 | if format == 'str': | |
| 104 | self.__formated_value = str(value) |
|
55 | self.__formated_value = str(value) | |
| 105 | return self.__formated_value |
|
56 | return self.__formated_value | |
| @@ -217,11 +168,9 class ParameterConf(): | |||||
| 217 | self.id = str(new_id) |
|
168 | self.id = str(new_id) | |
| 218 |
|
169 | |||
| 219 | def setup(self, id, name, value, format='str'): |
|
170 | def setup(self, id, name, value, format='str'): | |
|
|
171 | ||||
| 220 | self.id = str(id) |
|
172 | self.id = str(id) | |
| 221 | self.name = name |
|
173 | self.name = name | |
| 222 | if format == 'obj': |
|
|||
| 223 | self.value = value |
|
|||
| 224 | else: |
|
|||
| 225 |
|
|
174 | self.value = str(value) | |
| 226 | self.format = str.lower(format) |
|
175 | self.format = str.lower(format) | |
| 227 |
|
176 | |||
| @@ -236,7 +185,7 class ParameterConf(): | |||||
| 236 | self.format = format |
|
185 | self.format = format | |
| 237 |
|
186 | |||
| 238 | def makeXml(self, opElement): |
|
187 | def makeXml(self, opElement): | |
| 239 | if self.name not in ('queue',): |
|
188 | ||
| 240 |
|
|
189 | parmElement = SubElement(opElement, self.ELEMENTNAME) | |
| 241 |
|
|
190 | parmElement.set('id', str(self.id)) | |
| 242 |
|
|
191 | parmElement.set('name', self.name) | |
| @@ -334,19 +283,6 class OperationConf(): | |||||
| 334 |
|
283 | |||
| 335 | return value |
|
284 | return value | |
| 336 |
|
285 | |||
| 337 |
|
||||
| 338 | def getKwargs(self): |
|
|||
| 339 |
|
||||
| 340 | kwargs = {} |
|
|||
| 341 |
|
||||
| 342 | for parmConfObj in self.parmConfObjList: |
|
|||
| 343 | if self.name == 'run' and parmConfObj.name == 'datatype': |
|
|||
| 344 | continue |
|
|||
| 345 |
|
||||
| 346 | kwargs[parmConfObj.name] = parmConfObj.getValue() |
|
|||
| 347 |
|
||||
| 348 | return kwargs |
|
|||
| 349 |
|
||||
| 350 | def setup(self, id, name, priority, type): |
|
286 | def setup(self, id, name, priority, type): | |
| 351 |
|
287 | |||
| 352 | self.id = str(id) |
|
288 | self.id = str(id) | |
| @@ -435,7 +371,6 class OperationConf(): | |||||
| 435 |
|
371 | |||
| 436 | def createObject(self, plotter_queue=None): |
|
372 | def createObject(self, plotter_queue=None): | |
| 437 |
|
373 | |||
| 438 |
|
||||
| 439 | if self.type == 'self': |
|
374 | if self.type == 'self': | |
| 440 | raise ValueError, "This operation type cannot be created" |
|
375 | raise ValueError, "This operation type cannot be created" | |
| 441 |
|
376 | |||
| @@ -447,15 +382,11 class OperationConf(): | |||||
| 447 | opObj = Plotter(self.name, plotter_queue) |
|
382 | opObj = Plotter(self.name, plotter_queue) | |
| 448 |
|
383 | |||
| 449 | if self.type == 'external' or self.type == 'other': |
|
384 | if self.type == 'external' or self.type == 'other': | |
| 450 |
|
||||
| 451 | className = eval(self.name) |
|
385 | className = eval(self.name) | |
| 452 | kwargs = self.getKwargs() |
|
386 | opObj = className() | |
| 453 |
|
||||
| 454 | opObj = className(**kwargs) |
|
|||
| 455 |
|
387 | |||
| 456 | return opObj |
|
388 | return opObj | |
| 457 |
|
389 | |||
| 458 |
|
||||
| 459 | class ProcUnitConf(): |
|
390 | class ProcUnitConf(): | |
| 460 |
|
391 | |||
| 461 | id = None |
|
392 | id = None | |
| @@ -655,32 +586,19 class ProcUnitConf(): | |||||
| 655 | for opConfObj in self.opConfObjList: |
|
586 | for opConfObj in self.opConfObjList: | |
| 656 | opConfObj.printattr() |
|
587 | opConfObj.printattr() | |
| 657 |
|
588 | |||
| 658 |
|
||||
| 659 | def getKwargs(self): |
|
|||
| 660 |
|
||||
| 661 | opObj = self.opConfObjList[0] |
|
|||
| 662 | kwargs = opObj.getKwargs() |
|
|||
| 663 |
|
||||
| 664 | return kwargs |
|
|||
| 665 |
|
||||
| 666 | def createObjects(self, plotter_queue=None): |
|
589 | def createObjects(self, plotter_queue=None): | |
| 667 |
|
590 | |||
| 668 | className = eval(self.name) |
|
591 | className = eval(self.name) | |
| 669 | kwargs = self.getKwargs() |
|
592 | procUnitObj = className() | |
| 670 | procUnitObj = className(**kwargs) |
|
|||
| 671 |
|
593 | |||
| 672 | for opConfObj in self.opConfObjList: |
|
594 | for opConfObj in self.opConfObjList: | |
| 673 |
|
595 | |||
| 674 |
if opConfObj.type=='self' |
|
596 | if opConfObj.type == 'self': | |
| 675 | continue |
|
|||
| 676 | elif opConfObj.type=='self': |
|
|||
| 677 | procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs()) |
|
|||
| 678 | continue |
|
597 | continue | |
| 679 |
|
598 | |||
| 680 | opObj = opConfObj.createObject(plotter_queue) |
|
599 | opObj = opConfObj.createObject(plotter_queue) | |
| 681 |
|
600 | |||
| 682 | self.opObjDict[opConfObj.id] = opObj |
|
601 | self.opObjDict[opConfObj.id] = opObj | |
| 683 |
|
||||
| 684 | procUnitObj.addOperation(opObj, opConfObj.id) |
|
602 | procUnitObj.addOperation(opObj, opConfObj.id) | |
| 685 |
|
603 | |||
| 686 | self.procUnitObj = procUnitObj |
|
604 | self.procUnitObj = procUnitObj | |
| @@ -756,8 +674,7 class ReadUnitConf(ProcUnitConf): | |||||
| 756 |
|
674 | |||
| 757 | return self.ELEMENTNAME |
|
675 | return self.ELEMENTNAME | |
| 758 |
|
676 | |||
| 759 |
def setup(self, id, name, datatype, path |
|
677 | def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, **kwargs): | |
| 760 | endTime="", parentId=None, queue=None, server=None, **kwargs): |
|
|||
| 761 |
|
678 | |||
| 762 | #Compatible with old signal chain version |
|
679 | #Compatible with old signal chain version | |
| 763 | if datatype==None and name==None: |
|
680 | if datatype==None and name==None: | |
| @@ -768,13 +685,14 class ReadUnitConf(ProcUnitConf): | |||||
| 768 | name = datatype |
|
685 | name = datatype | |
| 769 | else: |
|
686 | else: | |
| 770 | name = '%sReader' %(datatype) |
|
687 | name = '%sReader' %(datatype) | |
|
|
688 | ||||
| 771 | if datatype==None: |
|
689 | if datatype==None: | |
| 772 | datatype = name.replace('Reader','') |
|
690 | datatype = name.replace('Reader','') | |
| 773 |
|
691 | |||
| 774 | self.id = id |
|
692 | self.id = id | |
| 775 | self.name = name |
|
693 | self.name = name | |
| 776 | self.datatype = datatype |
|
694 | self.datatype = datatype | |
| 777 | if path != '': |
|
695 | ||
| 778 |
|
|
696 | self.path = os.path.abspath(path) | |
| 779 | self.startDate = startDate |
|
697 | self.startDate = startDate | |
| 780 | self.endDate = endDate |
|
698 | self.endDate = endDate | |
| @@ -783,8 +701,7 class ReadUnitConf(ProcUnitConf): | |||||
| 783 |
|
701 | |||
| 784 | self.inputId = '0' |
|
702 | self.inputId = '0' | |
| 785 | self.parentId = parentId |
|
703 | self.parentId = parentId | |
| 786 | self.queue = queue |
|
704 | ||
| 787 | self.server = server |
|
|||
| 788 | self.addRunOperation(**kwargs) |
|
705 | self.addRunOperation(**kwargs) | |
| 789 |
|
706 | |||
| 790 | def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs): |
|
707 | def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs): | |
| @@ -826,19 +743,15 class ReadUnitConf(ProcUnitConf): | |||||
| 826 |
|
743 | |||
| 827 | opObj = self.addOperation(name = 'run', optype = 'self') |
|
744 | opObj = self.addOperation(name = 'run', optype = 'self') | |
| 828 |
|
745 | |||
| 829 | if self.server is None: |
|
|||
| 830 |
|
|
746 | opObj.addParameter(name='datatype' , value=self.datatype, format='str') | |
| 831 |
|
|
747 | opObj.addParameter(name='path' , value=self.path, format='str') | |
| 832 |
|
|
748 | opObj.addParameter(name='startDate' , value=self.startDate, format='date') | |
| 833 |
|
|
749 | opObj.addParameter(name='endDate' , value=self.endDate, format='date') | |
| 834 |
|
|
750 | opObj.addParameter(name='startTime' , value=self.startTime, format='time') | |
| 835 |
|
|
751 | opObj.addParameter(name='endTime' , value=self.endTime, format='time') | |
| 836 | opObj.addParameter(name='queue' , value=self.queue, format='obj') |
|
752 | ||
| 837 |
|
|
753 | for key, value in kwargs.items(): | |
| 838 |
|
|
754 | opObj.addParameter(name=key, value=value, format=type(value).__name__) | |
| 839 | else: |
|
|||
| 840 | opObj.addParameter(name='server' , value=self.server, format='str') |
|
|||
| 841 |
|
||||
| 842 |
|
755 | |||
| 843 | return opObj |
|
756 | return opObj | |
| 844 |
|
757 | |||
| @@ -1316,9 +1229,66 class Project(): | |||||
| 1316 |
|
1229 | |||
| 1317 | print "Process finished" |
|
1230 | print "Process finished" | |
| 1318 |
|
1231 | |||
| 1319 |
def start(self |
|
1232 | def start(self): | |
|
|
1233 | ||||
|
|
1234 | self.writeXml() | |||
| 1320 |
|
1235 | |||
| 1321 | self.writeXml(filename) |
|
|||
| 1322 | self.createObjects() |
|
1236 | self.createObjects() | |
| 1323 | self.connectObjects() |
|
1237 | self.connectObjects() | |
| 1324 | self.run() |
|
1238 | self.run() | |
|
|
1239 | ||||
|
|
1240 | if __name__ == '__main__': | |||
|
|
1241 | ||||
|
|
1242 | desc = "Segundo Test" | |||
|
|
1243 | filename = "schain.xml" | |||
|
|
1244 | ||||
|
|
1245 | controllerObj = Project() | |||
|
|
1246 | ||||
|
|
1247 | controllerObj.setup(id = '191', name='test01', description=desc) | |||
|
|
1248 | ||||
|
|
1249 | readUnitConfObj = controllerObj.addReadUnit(datatype='Voltage', | |||
|
|
1250 | path='data/rawdata/', | |||
|
|
1251 | startDate='2011/01/01', | |||
|
|
1252 | endDate='2012/12/31', | |||
|
|
1253 | startTime='00:00:00', | |||
|
|
1254 | endTime='23:59:59', | |||
|
|
1255 | online=1, | |||
|
|
1256 | walk=1) | |||
|
|
1257 | ||||
|
|
1258 | procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage', inputId=readUnitConfObj.getId()) | |||
|
|
1259 | ||||
|
|
1260 | opObj10 = procUnitConfObj0.addOperation(name='selectChannels') | |||
|
|
1261 | opObj10.addParameter(name='channelList', value='3,4,5', format='intlist') | |||
|
|
1262 | ||||
|
|
1263 | opObj10 = procUnitConfObj0.addOperation(name='selectHeights') | |||
|
|
1264 | opObj10.addParameter(name='minHei', value='90', format='float') | |||
|
|
1265 | opObj10.addParameter(name='maxHei', value='180', format='float') | |||
|
|
1266 | ||||
|
|
1267 | opObj12 = procUnitConfObj0.addOperation(name='CohInt', optype='external') | |||
|
|
1268 | opObj12.addParameter(name='n', value='10', format='int') | |||
|
|
1269 | ||||
|
|
1270 | procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj0.getId()) | |||
|
|
1271 | procUnitConfObj1.addParameter(name='nFFTPoints', value='32', format='int') | |||
|
|
1272 | # procUnitConfObj1.addParameter(name='pairList', value='(0,1),(0,2),(1,2)', format='') | |||
|
|
1273 | ||||
|
|
1274 | ||||
|
|
1275 | opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external') | |||
|
|
1276 | opObj11.addParameter(name='idfigure', value='1', format='int') | |||
|
|
1277 | opObj11.addParameter(name='wintitle', value='SpectraPlot0', format='str') | |||
|
|
1278 | opObj11.addParameter(name='zmin', value='40', format='int') | |||
|
|
1279 | opObj11.addParameter(name='zmax', value='90', format='int') | |||
|
|
1280 | opObj11.addParameter(name='showprofile', value='1', format='int') | |||
|
|
1281 | ||||
|
|
1282 | print "Escribiendo el archivo XML" | |||
|
|
1283 | ||||
|
|
1284 | controllerObj.writeXml(filename) | |||
|
|
1285 | ||||
|
|
1286 | print "Leyendo el archivo XML" | |||
|
|
1287 | controllerObj.readXml(filename) | |||
|
|
1288 | #controllerObj.printattr() | |||
|
|
1289 | ||||
|
|
1290 | controllerObj.createObjects() | |||
|
|
1291 | controllerObj.connectObjects() | |||
|
|
1292 | controllerObj.run() | |||
|
|
1293 | ||||
|
|
1294 | No newline at end of file | |||
| @@ -1,1 +1,1 | |||||
| 1 |
<Project description="Claire" id="002" name="script02"><ReadUnit datatype="VoltageReader" id="21" inputId="0" name="VoltageReader"><Operation id="211" name="run" priority="1" type="self"><Parameter format="str" id="2111" name="datatype" value="VoltageReader" /><Parameter format="str" id="2112" name="path" value="/media/nanosat/0BDE10E00BDE10E0/CLAIRE" /><Parameter format="date" id="2113" name="startDate" value="2017/07/26" /><Parameter format="date" id="2114" name="endDate" value="2017/07/26" /><Parameter format="time" id="2115" name="startTime" value="15:00:00" /><Parameter format="time" id="2116" name="endTime" value="16:00:00" /><Parameter format="int" id="2118" name=" |
|
1 | <Project description="Claire" id="002" name="script02"><ReadUnit datatype="VoltageReader" id="21" inputId="0" name="VoltageReader"><Operation id="211" name="run" priority="1" type="self"><Parameter format="str" id="2111" name="datatype" value="VoltageReader" /><Parameter format="str" id="2112" name="path" value="/media/nanosat/0BDE10E00BDE10E0/CLAIRE" /><Parameter format="date" id="2113" name="startDate" value="2017/07/26" /><Parameter format="date" id="2114" name="endDate" value="2017/07/26" /><Parameter format="time" id="2115" name="startTime" value="15:00:00" /><Parameter format="time" id="2116" name="endTime" value="16:00:00" /><Parameter format="int" id="2117" name="delay" value="30" /><Parameter format="int" id="2118" name="walk" value="1" /><Parameter format="int" id="2119" name="online" value="0" /></Operation><Operation id="212" name="printNumberOfBlock" priority="2" type="self" /></ReadUnit><ProcUnit datatype="VoltageProc" id="22" inputId="21" name="VoltageProc"><Operation id="221" name="run" priority="1" type="self" /></ProcUnit><ProcUnit datatype="SpectraProc" id="23" inputId="22" name="SpectraProc"><Operation id="231" name="run" priority="1" type="self"><Parameter format="int" id="2311" name="nFFTPoints" value="128" /><Parameter format="int" id="2312" name="nProfiles" value="128" /><Parameter format="pairslist" id="2313" name="pairsList" value="(0,1),(0,2),(1,2)" /></Operation><Operation id="232" name="setRadarFrequency" priority="2" type="self"><Parameter format="float" id="2321" name="frequency" value="445000000.0" /></Operation><Operation id="233" name="IncohInt" priority="3" type="other"><Parameter format="float" id="2331" name="timeInterval" value="128" /></Operation><Operation id="234" name="removeDC" priority="4" type="self"><Parameter format="int" id="2341" name="mode" value="2" /></Operation><Operation id="235" name="SpectraPlot" priority="5" type="external"><Parameter format="int" id="2351" name="id" value="235" /><Parameter format="str" id="2352" name="wintitle" value="SpectraPlot" /><Parameter format="str" id="2353" name="xaxis" value="frequency" /><Parameter format="float" id="2354" name="zmin" value="15.0" /><Parameter format="float" id="2355" name="zmax" value="50.0" /><Parameter format="int" id="2356" name="exp_code" value="21" /><Parameter format="int" id="2357" name="ftp_wei" value="0" /><Parameter format="int" id="2358" name="plot_pos" value="0" /></Operation></ProcUnit></Project> No newline at end of file | |
General Comments 0
You need to be logged in to leave comments.
Login now