##// END OF EJS Templates
schain-jc
RSS
controller.py
José Chávez -
r1060:70b07edd23c2
parent child
Show More
Side by Side Unified
Context file:
r1060:70b07edd23c2 Loading diff...:
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (834 lines changed) Show them Hide them
@@ -1,1280 +1,1324
1 '''
1 '''
2 Created on September , 2012
2 Created on September , 2012
3 @author:
3 @author:
4 '''
4 '''
5
5
6 import sys
6 import sys
7 import ast
7 import ast
8 import datetime
8 import datetime
9 import traceback
9 import traceback
10 import math
11 import time
12 from multiprocessing import Process, Queue, cpu_count
13
10 import schainpy
14 import schainpy
11 import schainpy.admin
15 import schainpy.admin
12 from schainpy.utils.log import logToFile
16 from schainpy.utils.log import logToFile
13
17
14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
18 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
15 from xml.dom import minidom
19 from xml.dom import minidom
16
20
17 from multiprocessing import cpu_count
18 from schainpy.model import *
21 from schainpy.model import *
19 from time import sleep
22 from time import sleep
20
23
21
24
25
22 def prettify(elem):
26 def prettify(elem):
23 """Return a pretty-printed XML string for the Element.
27 """Return a pretty-printed XML string for the Element.
24 """
28 """
25 rough_string = tostring(elem, 'utf-8')
29 rough_string = tostring(elem, 'utf-8')
26 reparsed = minidom.parseString(rough_string)
30 reparsed = minidom.parseString(rough_string)
27 return reparsed.toprettyxml(indent=" ")
31 return reparsed.toprettyxml(indent=" ")
28
32
29 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
33 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
30 skip = 0
34 skip = 0
31 cursor = 0
35 cursor = 0
32 nFiles = None
36 nFiles = None
33 processes = []
37 processes = []
34 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
38 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
35 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
39 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
36 days = (dt2 - dt1).days
40 days = (dt2 - dt1).days
37
41
38 for day in range(days+1):
42 for day in range(days+1):
39 skip = 0
43 skip = 0
40 cursor = 0
44 cursor = 0
41 q = Queue()
45 q = Queue()
42 processes = []
46 processes = []
43 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
47 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
44 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
48 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
45 firstProcess.start()
49 firstProcess.start()
46 if by_day:
50 if by_day:
47 continue
51 continue
48 nFiles = q.get()
52 nFiles = q.get()
49 if nFiles==0:
53 if nFiles==0:
50 continue
54 continue
51 firstProcess.terminate()
55 firstProcess.terminate()
52 skip = int(math.ceil(nFiles/nProcess))
56 skip = int(math.ceil(nFiles/nProcess))
53 while True:
57 while True:
54 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
58 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
55 processes[cursor].start()
59 processes[cursor].start()
56 if nFiles < cursor*skip:
60 if nFiles < cursor*skip:
57 break
61 break
58 cursor += 1
62 cursor += 1
59
63
60 def beforeExit(exctype, value, trace):
64 def beforeExit(exctype, value, trace):
61 for process in processes:
65 for process in processes:
62 process.terminate()
66 process.terminate()
63 process.join()
67 process.join()
64 print traceback.print_tb(trace)
68 print traceback.print_tb(trace)
65
69
66 sys.excepthook = beforeExit
70 sys.excepthook = beforeExit
67
71
68 for process in processes:
72 for process in processes:
69 process.join()
73 process.join()
70 process.terminate()
74 process.terminate()
71
75
72 time.sleep(3)
76 time.sleep(3)
73
77
74
78
75 class ParameterConf():
79 class ParameterConf():
76
80
77 id = None
81 id = None
78 name = None
82 name = None
79 value = None
83 value = None
80 format = None
84 format = None
81
85
82 __formated_value = None
86 __formated_value = None
83
87
84 ELEMENTNAME = 'Parameter'
88 ELEMENTNAME = 'Parameter'
85
89
86 def __init__(self):
90 def __init__(self):
87
91
88 self.format = 'str'
92 self.format = 'str'
89
93
90 def getElementName(self):
94 def getElementName(self):
91
95
92 return self.ELEMENTNAME
96 return self.ELEMENTNAME
93
97
94 def getValue(self):
98 def getValue(self):
95
99
96 value = self.value
100 value = self.value
97 format = self.format
101 format = self.format
98
102
99 if self.__formated_value != None:
103 if self.__formated_value != None:
100
104
101 return self.__formated_value
105 return self.__formated_value
102
106
107 if format == 'obj':
108 return value
109
103 if format == 'str':
110 if format == 'str':
104 self.__formated_value = str(value)
111 self.__formated_value = str(value)
105 return self.__formated_value
112 return self.__formated_value
106
113
107 if value == '':
114 if value == '':
108 raise ValueError, "%s: This parameter value is empty" %self.name
115 raise ValueError, "%s: This parameter value is empty" %self.name
109
116
110 if format == 'list':
117 if format == 'list':
111 strList = value.split(',')
118 strList = value.split(',')
112
119
113 self.__formated_value = strList
120 self.__formated_value = strList
114
121
115 return self.__formated_value
122 return self.__formated_value
116
123
117 if format == 'intlist':
124 if format == 'intlist':
118 """
125 """
119 Example:
126 Example:
120 value = (0,1,2)
127 value = (0,1,2)
121 """
128 """
122
129
123 new_value = ast.literal_eval(value)
130 new_value = ast.literal_eval(value)
124
131
125 if type(new_value) not in (tuple, list):
132 if type(new_value) not in (tuple, list):
126 new_value = [int(new_value)]
133 new_value = [int(new_value)]
127
134
128 self.__formated_value = new_value
135 self.__formated_value = new_value
129
136
130 return self.__formated_value
137 return self.__formated_value
131
138
132 if format == 'floatlist':
139 if format == 'floatlist':
133 """
140 """
134 Example:
141 Example:
135 value = (0.5, 1.4, 2.7)
142 value = (0.5, 1.4, 2.7)
136 """
143 """
137
144
138 new_value = ast.literal_eval(value)
145 new_value = ast.literal_eval(value)
139
146
140 if type(new_value) not in (tuple, list):
147 if type(new_value) not in (tuple, list):
141 new_value = [float(new_value)]
148 new_value = [float(new_value)]
142
149
143 self.__formated_value = new_value
150 self.__formated_value = new_value
144
151
145 return self.__formated_value
152 return self.__formated_value
146
153
147 if format == 'date':
154 if format == 'date':
148 strList = value.split('/')
155 strList = value.split('/')
149 intList = [int(x) for x in strList]
156 intList = [int(x) for x in strList]
150 date = datetime.date(intList[0], intList[1], intList[2])
157 date = datetime.date(intList[0], intList[1], intList[2])
151
158
152 self.__formated_value = date
159 self.__formated_value = date
153
160
154 return self.__formated_value
161 return self.__formated_value
155
162
156 if format == 'time':
163 if format == 'time':
157 strList = value.split(':')
164 strList = value.split(':')
158 intList = [int(x) for x in strList]
165 intList = [int(x) for x in strList]
159 time = datetime.time(intList[0], intList[1], intList[2])
166 time = datetime.time(intList[0], intList[1], intList[2])
160
167
161 self.__formated_value = time
168 self.__formated_value = time
162
169
163 return self.__formated_value
170 return self.__formated_value
164
171
165 if format == 'pairslist':
172 if format == 'pairslist':
166 """
173 """
167 Example:
174 Example:
168 value = (0,1),(1,2)
175 value = (0,1),(1,2)
169 """
176 """
170
177
171 new_value = ast.literal_eval(value)
178 new_value = ast.literal_eval(value)
172
179
173 if type(new_value) not in (tuple, list):
180 if type(new_value) not in (tuple, list):
174 raise ValueError, "%s has to be a tuple or list of pairs" %value
181 raise ValueError, "%s has to be a tuple or list of pairs" %value
175
182
176 if type(new_value[0]) not in (tuple, list):
183 if type(new_value[0]) not in (tuple, list):
177 if len(new_value) != 2:
184 if len(new_value) != 2:
178 raise ValueError, "%s has to be a tuple or list of pairs" %value
185 raise ValueError, "%s has to be a tuple or list of pairs" %value
179 new_value = [new_value]
186 new_value = [new_value]
180
187
181 for thisPair in new_value:
188 for thisPair in new_value:
182 if len(thisPair) != 2:
189 if len(thisPair) != 2:
183 raise ValueError, "%s has to be a tuple or list of pairs" %value
190 raise ValueError, "%s has to be a tuple or list of pairs" %value
184
191
185 self.__formated_value = new_value
192 self.__formated_value = new_value
186
193
187 return self.__formated_value
194 return self.__formated_value
188
195
189 if format == 'multilist':
196 if format == 'multilist':
190 """
197 """
191 Example:
198 Example:
192 value = (0,1,2),(3,4,5)
199 value = (0,1,2),(3,4,5)
193 """
200 """
194 multiList = ast.literal_eval(value)
201 multiList = ast.literal_eval(value)
195
202
196 if type(multiList[0]) == int:
203 if type(multiList[0]) == int:
197 multiList = ast.literal_eval("(" + value + ")")
204 multiList = ast.literal_eval("(" + value + ")")
198
205
199 self.__formated_value = multiList
206 self.__formated_value = multiList
200
207
201 return self.__formated_value
208 return self.__formated_value
202
209
203 if format == 'bool':
210 if format == 'bool':
204 value = int(value)
211 value = int(value)
205
212
206 if format == 'int':
213 if format == 'int':
207 value = float(value)
214 value = float(value)
208
215
209 format_func = eval(format)
216 format_func = eval(format)
210
217
211 self.__formated_value = format_func(value)
218 self.__formated_value = format_func(value)
212
219
213 return self.__formated_value
220 return self.__formated_value
214
221
215 def updateId(self, new_id):
222 def updateId(self, new_id):
216
223
217 self.id = str(new_id)
224 self.id = str(new_id)
218
225
219 def setup(self, id, name, value, format='str'):
226 def setup(self, id, name, value, format='str'):
220
221 self.id = str(id)
227 self.id = str(id)
222 self.name = name
228 self.name = name
223 self.value = str(value)
229 if format == 'obj':
230 self.value = value
231 else:
232 self.value = str(value)
224 self.format = str.lower(format)
233 self.format = str.lower(format)
225
234
226 self.getValue()
235 self.getValue()
227
236
228 return 1
237 return 1
229
238
230 def update(self, name, value, format='str'):
239 def update(self, name, value, format='str'):
231
240
232 self.name = name
241 self.name = name
233 self.value = str(value)
242 self.value = str(value)
234 self.format = format
243 self.format = format
235
244
236 def makeXml(self, opElement):
245 def makeXml(self, opElement):
237
246 if self.name not in ('queue',):
238 parmElement = SubElement(opElement, self.ELEMENTNAME)
247 parmElement = SubElement(opElement, self.ELEMENTNAME)
239 parmElement.set('id', str(self.id))
248 parmElement.set('id', str(self.id))
240 parmElement.set('name', self.name)
249 parmElement.set('name', self.name)
241 parmElement.set('value', self.value)
250 parmElement.set('value', self.value)
242 parmElement.set('format', self.format)
251 parmElement.set('format', self.format)
243
252
244 def readXml(self, parmElement):
253 def readXml(self, parmElement):
245
254
246 self.id = parmElement.get('id')
255 self.id = parmElement.get('id')
247 self.name = parmElement.get('name')
256 self.name = parmElement.get('name')
248 self.value = parmElement.get('value')
257 self.value = parmElement.get('value')
249 self.format = str.lower(parmElement.get('format'))
258 self.format = str.lower(parmElement.get('format'))
250
259
251 #Compatible with old signal chain version
260 #Compatible with old signal chain version
252 if self.format == 'int' and self.name == 'idfigure':
261 if self.format == 'int' and self.name == 'idfigure':
253 self.name = 'id'
262 self.name = 'id'
254
263
255 def printattr(self):
264 def printattr(self):
256
265
257 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
266 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
258
267
259 class OperationConf():
268 class OperationConf():
260
269
261 id = None
270 id = None
262 name = None
271 name = None
263 priority = None
272 priority = None
264 type = None
273 type = None
265
274
266 parmConfObjList = []
275 parmConfObjList = []
267
276
268 ELEMENTNAME = 'Operation'
277 ELEMENTNAME = 'Operation'
269
278
270 def __init__(self):
279 def __init__(self):
271
280
272 self.id = '0'
281 self.id = '0'
273 self.name = None
282 self.name = None
274 self.priority = None
283 self.priority = None
275 self.type = 'self'
284 self.type = 'self'
276
285
277
286
278 def __getNewId(self):
287 def __getNewId(self):
279
288
280 return int(self.id)*10 + len(self.parmConfObjList) + 1
289 return int(self.id)*10 + len(self.parmConfObjList) + 1
281
290
282 def updateId(self, new_id):
291 def updateId(self, new_id):
283
292
284 self.id = str(new_id)
293 self.id = str(new_id)
285
294
286 n = 1
295 n = 1
287 for parmObj in self.parmConfObjList:
296 for parmObj in self.parmConfObjList:
288
297
289 idParm = str(int(new_id)*10 + n)
298 idParm = str(int(new_id)*10 + n)
290 parmObj.updateId(idParm)
299 parmObj.updateId(idParm)
291
300
292 n += 1
301 n += 1
293
302
294 def getElementName(self):
303 def getElementName(self):
295
304
296 return self.ELEMENTNAME
305 return self.ELEMENTNAME
297
306
298 def getParameterObjList(self):
307 def getParameterObjList(self):
299
308
300 return self.parmConfObjList
309 return self.parmConfObjList
301
310
302 def getParameterObj(self, parameterName):
311 def getParameterObj(self, parameterName):
303
312
304 for parmConfObj in self.parmConfObjList:
313 for parmConfObj in self.parmConfObjList:
305
314
306 if parmConfObj.name != parameterName:
315 if parmConfObj.name != parameterName:
307 continue
316 continue
308
317
309 return parmConfObj
318 return parmConfObj
310
319
311 return None
320 return None
312
321
313 def getParameterObjfromValue(self, parameterValue):
322 def getParameterObjfromValue(self, parameterValue):
314
323
315 for parmConfObj in self.parmConfObjList:
324 for parmConfObj in self.parmConfObjList:
316
325
317 if parmConfObj.getValue() != parameterValue:
326 if parmConfObj.getValue() != parameterValue:
318 continue
327 continue
319
328
320 return parmConfObj.getValue()
329 return parmConfObj.getValue()
321
330
322 return None
331 return None
323
332
324 def getParameterValue(self, parameterName):
333 def getParameterValue(self, parameterName):
325
334
326 parameterObj = self.getParameterObj(parameterName)
335 parameterObj = self.getParameterObj(parameterName)
327
336
328 # if not parameterObj:
337 # if not parameterObj:
329 # return None
338 # return None
330
339
331 value = parameterObj.getValue()
340 value = parameterObj.getValue()
332
341
333 return value
342 return value
334
343
344
345 def getKwargs(self):
346
347 kwargs = {}
348
349 for parmConfObj in self.parmConfObjList:
350 if self.name == 'run' and parmConfObj.name == 'datatype':
351 continue
352
353 kwargs[parmConfObj.name] = parmConfObj.getValue()
354
355 return kwargs
356
335 def setup(self, id, name, priority, type):
357 def setup(self, id, name, priority, type):
336
358
337 self.id = str(id)
359 self.id = str(id)
338 self.name = name
360 self.name = name
339 self.type = type
361 self.type = type
340 self.priority = priority
362 self.priority = priority
341
363
342 self.parmConfObjList = []
364 self.parmConfObjList = []
343
365
344 def removeParameters(self):
366 def removeParameters(self):
345
367
346 for obj in self.parmConfObjList:
368 for obj in self.parmConfObjList:
347 del obj
369 del obj
348
370
349 self.parmConfObjList = []
371 self.parmConfObjList = []
350
372
351 def addParameter(self, name, value, format='str'):
373 def addParameter(self, name, value, format='str'):
352
374
353 id = self.__getNewId()
375 id = self.__getNewId()
354
376
355 parmConfObj = ParameterConf()
377 parmConfObj = ParameterConf()
356 if not parmConfObj.setup(id, name, value, format):
378 if not parmConfObj.setup(id, name, value, format):
357 return None
379 return None
358
380
359 self.parmConfObjList.append(parmConfObj)
381 self.parmConfObjList.append(parmConfObj)
360
382
361 return parmConfObj
383 return parmConfObj
362
384
363 def changeParameter(self, name, value, format='str'):
385 def changeParameter(self, name, value, format='str'):
364
386
365 parmConfObj = self.getParameterObj(name)
387 parmConfObj = self.getParameterObj(name)
366 parmConfObj.update(name, value, format)
388 parmConfObj.update(name, value, format)
367
389
368 return parmConfObj
390 return parmConfObj
369
391
370 def makeXml(self, procUnitElement):
392 def makeXml(self, procUnitElement):
371
393
372 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
394 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
373 opElement.set('id', str(self.id))
395 opElement.set('id', str(self.id))
374 opElement.set('name', self.name)
396 opElement.set('name', self.name)
375 opElement.set('type', self.type)
397 opElement.set('type', self.type)
376 opElement.set('priority', str(self.priority))
398 opElement.set('priority', str(self.priority))
377
399
378 for parmConfObj in self.parmConfObjList:
400 for parmConfObj in self.parmConfObjList:
379 parmConfObj.makeXml(opElement)
401 parmConfObj.makeXml(opElement)
380
402
381 def readXml(self, opElement):
403 def readXml(self, opElement):
382
404
383 self.id = opElement.get('id')
405 self.id = opElement.get('id')
384 self.name = opElement.get('name')
406 self.name = opElement.get('name')
385 self.type = opElement.get('type')
407 self.type = opElement.get('type')
386 self.priority = opElement.get('priority')
408 self.priority = opElement.get('priority')
387
409
388 #Compatible with old signal chain version
410 #Compatible with old signal chain version
389 #Use of 'run' method instead 'init'
411 #Use of 'run' method instead 'init'
390 if self.type == 'self' and self.name == 'init':
412 if self.type == 'self' and self.name == 'init':
391 self.name = 'run'
413 self.name = 'run'
392
414
393 self.parmConfObjList = []
415 self.parmConfObjList = []
394
416
395 parmElementList = opElement.iter(ParameterConf().getElementName())
417 parmElementList = opElement.iter(ParameterConf().getElementName())
396
418
397 for parmElement in parmElementList:
419 for parmElement in parmElementList:
398 parmConfObj = ParameterConf()
420 parmConfObj = ParameterConf()
399 parmConfObj.readXml(parmElement)
421 parmConfObj.readXml(parmElement)
400
422
401 #Compatible with old signal chain version
423 #Compatible with old signal chain version
402 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
424 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
403 if self.type != 'self' and self.name == 'Plot':
425 if self.type != 'self' and self.name == 'Plot':
404 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
426 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
405 self.name = parmConfObj.value
427 self.name = parmConfObj.value
406 continue
428 continue
407
429
408 self.parmConfObjList.append(parmConfObj)
430 self.parmConfObjList.append(parmConfObj)
409
431
410 def printattr(self):
432 def printattr(self):
411
433
412 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
434 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
413 self.id,
435 self.id,
414 self.name,
436 self.name,
415 self.type,
437 self.type,
416 self.priority)
438 self.priority)
417
439
418 for parmConfObj in self.parmConfObjList:
440 for parmConfObj in self.parmConfObjList:
419 parmConfObj.printattr()
441 parmConfObj.printattr()
420
442
421 def createObject(self, plotter_queue=None):
443 def createObject(self, plotter_queue=None):
422
444
445
423 if self.type == 'self':
446 if self.type == 'self':
424 raise ValueError, "This operation type cannot be created"
447 raise ValueError, "This operation type cannot be created"
425
448
426 if self.type == 'plotter':
449 if self.type == 'plotter':
427 #Plotter(plotter_name)
450 #Plotter(plotter_name)
428 if not plotter_queue:
451 if not plotter_queue:
429 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
452 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
430
453
431 opObj = Plotter(self.name, plotter_queue)
454 opObj = Plotter(self.name, plotter_queue)
432
455
433 if self.type == 'external' or self.type == 'other':
456 if self.type == 'external' or self.type == 'other':
457
434 className = eval(self.name)
458 className = eval(self.name)
435 opObj = className()
459 kwargs = self.getKwargs()
436
460
461 opObj = className(**kwargs)
462
437 return opObj
463 return opObj
438
464
465
439 class ProcUnitConf():
466 class ProcUnitConf():
440
467
441 id = None
468 id = None
442 name = None
469 name = None
443 datatype = None
470 datatype = None
444 inputId = None
471 inputId = None
445 parentId = None
472 parentId = None
446
473
447 opConfObjList = []
474 opConfObjList = []
448
475
449 procUnitObj = None
476 procUnitObj = None
450 opObjList = []
477 opObjList = []
451
478
452 ELEMENTNAME = 'ProcUnit'
479 ELEMENTNAME = 'ProcUnit'
453
480
454 def __init__(self):
481 def __init__(self):
455
482
456 self.id = None
483 self.id = None
457 self.datatype = None
484 self.datatype = None
458 self.name = None
485 self.name = None
459 self.inputId = None
486 self.inputId = None
460
487
461 self.opConfObjList = []
488 self.opConfObjList = []
462
489
463 self.procUnitObj = None
490 self.procUnitObj = None
464 self.opObjDict = {}
491 self.opObjDict = {}
465
492
466 def __getPriority(self):
493 def __getPriority(self):
467
494
468 return len(self.opConfObjList)+1
495 return len(self.opConfObjList)+1
469
496
470 def __getNewId(self):
497 def __getNewId(self):
471
498
472 return int(self.id)*10 + len(self.opConfObjList) + 1
499 return int(self.id)*10 + len(self.opConfObjList) + 1
473
500
474 def getElementName(self):
501 def getElementName(self):
475
502
476 return self.ELEMENTNAME
503 return self.ELEMENTNAME
477
504
478 def getId(self):
505 def getId(self):
479
506
480 return self.id
507 return self.id
481
508
482 def updateId(self, new_id, parentId=parentId):
509 def updateId(self, new_id, parentId=parentId):
483
510
484
511
485 new_id = int(parentId)*10 + (int(self.id) % 10)
512 new_id = int(parentId)*10 + (int(self.id) % 10)
486 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
513 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
487
514
488 #If this proc unit has not inputs
515 #If this proc unit has not inputs
489 if self.inputId == '0':
516 if self.inputId == '0':
490 new_inputId = 0
517 new_inputId = 0
491
518
492 n = 1
519 n = 1
493 for opConfObj in self.opConfObjList:
520 for opConfObj in self.opConfObjList:
494
521
495 idOp = str(int(new_id)*10 + n)
522 idOp = str(int(new_id)*10 + n)
496 opConfObj.updateId(idOp)
523 opConfObj.updateId(idOp)
497
524
498 n += 1
525 n += 1
499
526
500 self.parentId = str(parentId)
527 self.parentId = str(parentId)
501 self.id = str(new_id)
528 self.id = str(new_id)
502 self.inputId = str(new_inputId)
529 self.inputId = str(new_inputId)
503
530
504
531
505 def getInputId(self):
532 def getInputId(self):
506
533
507 return self.inputId
534 return self.inputId
508
535
509 def getOperationObjList(self):
536 def getOperationObjList(self):
510
537
511 return self.opConfObjList
538 return self.opConfObjList
512
539
513 def getOperationObj(self, name=None):
540 def getOperationObj(self, name=None):
514
541
515 for opConfObj in self.opConfObjList:
542 for opConfObj in self.opConfObjList:
516
543
517 if opConfObj.name != name:
544 if opConfObj.name != name:
518 continue
545 continue
519
546
520 return opConfObj
547 return opConfObj
521
548
522 return None
549 return None
523
550
524 def getOpObjfromParamValue(self, value=None):
551 def getOpObjfromParamValue(self, value=None):
525
552
526 for opConfObj in self.opConfObjList:
553 for opConfObj in self.opConfObjList:
527 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
554 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
528 continue
555 continue
529 return opConfObj
556 return opConfObj
530 return None
557 return None
531
558
532 def getProcUnitObj(self):
559 def getProcUnitObj(self):
533
560
534 return self.procUnitObj
561 return self.procUnitObj
535
562
536 def setup(self, id, name, datatype, inputId, parentId=None):
563 def setup(self, id, name, datatype, inputId, parentId=None):
537
564
538 #Compatible with old signal chain version
565 #Compatible with old signal chain version
539 if datatype==None and name==None:
566 if datatype==None and name==None:
540 raise ValueError, "datatype or name should be defined"
567 raise ValueError, "datatype or name should be defined"
541
568
542 if name==None:
569 if name==None:
543 if 'Proc' in datatype:
570 if 'Proc' in datatype:
544 name = datatype
571 name = datatype
545 else:
572 else:
546 name = '%sProc' %(datatype)
573 name = '%sProc' %(datatype)
547
574
548 if datatype==None:
575 if datatype==None:
549 datatype = name.replace('Proc','')
576 datatype = name.replace('Proc','')
550
577
551 self.id = str(id)
578 self.id = str(id)
552 self.name = name
579 self.name = name
553 self.datatype = datatype
580 self.datatype = datatype
554 self.inputId = inputId
581 self.inputId = inputId
555 self.parentId = parentId
582 self.parentId = parentId
556
583
557 self.opConfObjList = []
584 self.opConfObjList = []
558
585
559 self.addOperation(name='run', optype='self')
586 self.addOperation(name='run', optype='self')
560
587
561 def removeOperations(self):
588 def removeOperations(self):
562
589
563 for obj in self.opConfObjList:
590 for obj in self.opConfObjList:
564 del obj
591 del obj
565
592
566 self.opConfObjList = []
593 self.opConfObjList = []
567 self.addOperation(name='run')
594 self.addOperation(name='run')
568
595
569 def addParameter(self, **kwargs):
596 def addParameter(self, **kwargs):
570 '''
597 '''
571 Add parameters to "run" operation
598 Add parameters to "run" operation
572 '''
599 '''
573 opObj = self.opConfObjList[0]
600 opObj = self.opConfObjList[0]
574
601
575 opObj.addParameter(**kwargs)
602 opObj.addParameter(**kwargs)
576
603
577 return opObj
604 return opObj
578
605
579 def addOperation(self, name, optype='self'):
606 def addOperation(self, name, optype='self'):
580
607
581 id = self.__getNewId()
608 id = self.__getNewId()
582 priority = self.__getPriority()
609 priority = self.__getPriority()
583
610
584 opConfObj = OperationConf()
611 opConfObj = OperationConf()
585 opConfObj.setup(id, name=name, priority=priority, type=optype)
612 opConfObj.setup(id, name=name, priority=priority, type=optype)
586
613
587 self.opConfObjList.append(opConfObj)
614 self.opConfObjList.append(opConfObj)
588
615
589 return opConfObj
616 return opConfObj
590
617
591 def makeXml(self, projectElement):
618 def makeXml(self, projectElement):
592
619
593 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
620 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
594 procUnitElement.set('id', str(self.id))
621 procUnitElement.set('id', str(self.id))
595 procUnitElement.set('name', self.name)
622 procUnitElement.set('name', self.name)
596 procUnitElement.set('datatype', self.datatype)
623 procUnitElement.set('datatype', self.datatype)
597 procUnitElement.set('inputId', str(self.inputId))
624 procUnitElement.set('inputId', str(self.inputId))
598
625
599 for opConfObj in self.opConfObjList:
626 for opConfObj in self.opConfObjList:
600 opConfObj.makeXml(procUnitElement)
627 opConfObj.makeXml(procUnitElement)
601
628
602 def readXml(self, upElement):
629 def readXml(self, upElement):
603
630
604 self.id = upElement.get('id')
631 self.id = upElement.get('id')
605 self.name = upElement.get('name')
632 self.name = upElement.get('name')
606 self.datatype = upElement.get('datatype')
633 self.datatype = upElement.get('datatype')
607 self.inputId = upElement.get('inputId')
634 self.inputId = upElement.get('inputId')
608
635
609 if self.ELEMENTNAME == "ReadUnit":
636 if self.ELEMENTNAME == "ReadUnit":
610 self.datatype = self.datatype.replace("Reader", "")
637 self.datatype = self.datatype.replace("Reader", "")
611
638
612 if self.ELEMENTNAME == "ProcUnit":
639 if self.ELEMENTNAME == "ProcUnit":
613 self.datatype = self.datatype.replace("Proc", "")
640 self.datatype = self.datatype.replace("Proc", "")
614
641
615 if self.inputId == 'None':
642 if self.inputId == 'None':
616 self.inputId = '0'
643 self.inputId = '0'
617
644
618 self.opConfObjList = []
645 self.opConfObjList = []
619
646
620 opElementList = upElement.iter(OperationConf().getElementName())
647 opElementList = upElement.iter(OperationConf().getElementName())
621
648
622 for opElement in opElementList:
649 for opElement in opElementList:
623 opConfObj = OperationConf()
650 opConfObj = OperationConf()
624 opConfObj.readXml(opElement)
651 opConfObj.readXml(opElement)
625 self.opConfObjList.append(opConfObj)
652 self.opConfObjList.append(opConfObj)
626
653
627 def printattr(self):
654 def printattr(self):
628
655
629 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
656 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
630 self.id,
657 self.id,
631 self.name,
658 self.name,
632 self.datatype,
659 self.datatype,
633 self.inputId)
660 self.inputId)
634
661
635 for opConfObj in self.opConfObjList:
662 for opConfObj in self.opConfObjList:
636 opConfObj.printattr()
663 opConfObj.printattr()
637
664
665
666 def getKwargs(self):
667
668 opObj = self.opConfObjList[0]
669 kwargs = opObj.getKwargs()
670
671 return kwargs
672
638 def createObjects(self, plotter_queue=None):
673 def createObjects(self, plotter_queue=None):
639
674
640 className = eval(self.name)
675 className = eval(self.name)
641 procUnitObj = className()
676 kwargs = self.getKwargs()
642
677 procUnitObj = className(**kwargs)
678
643 for opConfObj in self.opConfObjList:
679 for opConfObj in self.opConfObjList:
644
680
645 if opConfObj.type == 'self':
681 if opConfObj.type=='self' and self.name=='run':
646 continue
682 continue
647
683 elif opConfObj.type=='self':
684 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
685 continue
686
648 opObj = opConfObj.createObject(plotter_queue)
687 opObj = opConfObj.createObject(plotter_queue)
649
688
650 self.opObjDict[opConfObj.id] = opObj
689 self.opObjDict[opConfObj.id] = opObj
690
651 procUnitObj.addOperation(opObj, opConfObj.id)
691 procUnitObj.addOperation(opObj, opConfObj.id)
652
692
653 self.procUnitObj = procUnitObj
693 self.procUnitObj = procUnitObj
654
694
655 return procUnitObj
695 return procUnitObj
656
696
657 def run(self):
697 def run(self):
658
698
659 is_ok = False
699 is_ok = False
660
700
661 for opConfObj in self.opConfObjList:
701 for opConfObj in self.opConfObjList:
662
702
663 kwargs = {}
703 kwargs = {}
664 for parmConfObj in opConfObj.getParameterObjList():
704 for parmConfObj in opConfObj.getParameterObjList():
665 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
705 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
666 continue
706 continue
667
707
668 kwargs[parmConfObj.name] = parmConfObj.getValue()
708 kwargs[parmConfObj.name] = parmConfObj.getValue()
669
709
670 #ini = time.time()
710 #ini = time.time()
671
711
672 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
712 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
673 sts = self.procUnitObj.call(opType = opConfObj.type,
713 sts = self.procUnitObj.call(opType = opConfObj.type,
674 opName = opConfObj.name,
714 opName = opConfObj.name,
675 opId = opConfObj.id)
715 opId = opConfObj.id)
676
716
677 # total_time = time.time() - ini
717 # total_time = time.time() - ini
678 #
718 #
679 # if total_time > 0.002:
719 # if total_time > 0.002:
680 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
720 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
681
721
682 is_ok = is_ok or sts
722 is_ok = is_ok or sts
683
723
684 return is_ok
724 return is_ok
685
725
686 def close(self):
726 def close(self):
687
727
688 for opConfObj in self.opConfObjList:
728 for opConfObj in self.opConfObjList:
689 if opConfObj.type == 'self':
729 if opConfObj.type == 'self':
690 continue
730 continue
691
731
692 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
732 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
693 opObj.close()
733 opObj.close()
694
734
695 self.procUnitObj.close()
735 self.procUnitObj.close()
696
736
697 return
737 return
698
738
699 class ReadUnitConf(ProcUnitConf):
739 class ReadUnitConf(ProcUnitConf):
700
740
701 path = None
741 path = None
702 startDate = None
742 startDate = None
703 endDate = None
743 endDate = None
704 startTime = None
744 startTime = None
705 endTime = None
745 endTime = None
706
746
707 ELEMENTNAME = 'ReadUnit'
747 ELEMENTNAME = 'ReadUnit'
708
748
709 def __init__(self):
749 def __init__(self):
710
750
711 self.id = None
751 self.id = None
712 self.datatype = None
752 self.datatype = None
713 self.name = None
753 self.name = None
714 self.inputId = None
754 self.inputId = None
715
755
716 self.parentId = None
756 self.parentId = None
717
757
718 self.opConfObjList = []
758 self.opConfObjList = []
719 self.opObjList = []
759 self.opObjList = []
720
760
721 def getElementName(self):
761 def getElementName(self):
722
762
723 return self.ELEMENTNAME
763 return self.ELEMENTNAME
724
764
725 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
765 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
726 endTime="", parentId=None, queue=None, server=None, **kwargs):
766 endTime="", parentId=None, queue=None, server=None, **kwargs):
727 #Compatible with old signal chain version
767 #Compatible with old signal chain version
728 if datatype==None and name==None:
768 if datatype==None and name==None:
729 raise ValueError, "datatype or name should be defined"
769 raise ValueError, "datatype or name should be defined"
730
770
731 if name==None:
771 if name==None:
732 if 'Reader' in datatype:
772 if 'Reader' in datatype:
733 name = datatype
773 name = datatype
734 else:
774 else:
735 name = '%sReader' %(datatype)
775 name = '%sReader' %(datatype)
736
737 if datatype==None:
776 if datatype==None:
738 datatype = name.replace('Reader','')
777 datatype = name.replace('Reader','')
739
778
740 self.id = id
779 self.id = id
741 self.name = name
780 self.name = name
742 self.datatype = datatype
781 self.datatype = datatype
743
782 if path != '':
744 self.path = os.path.abspath(path)
783 self.path = os.path.abspath(path)
745 self.startDate = startDate
784 self.startDate = startDate
746 self.endDate = endDate
785 self.endDate = endDate
747 self.startTime = startTime
786 self.startTime = startTime
748 self.endTime = endTime
787 self.endTime = endTime
749
788
750 self.inputId = '0'
789 self.inputId = '0'
751 self.parentId = parentId
790 self.parentId = parentId
752
791 self.queue = queue
792 self.server = server
753 self.addRunOperation(**kwargs)
793 self.addRunOperation(**kwargs)
754
794
755 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
795 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
756
796
757 #Compatible with old signal chain version
797 #Compatible with old signal chain version
758 if datatype==None and name==None:
798 if datatype==None and name==None:
759 raise ValueError, "datatype or name should be defined"
799 raise ValueError, "datatype or name should be defined"
760
800
761 if name==None:
801 if name==None:
762 if 'Reader' in datatype:
802 if 'Reader' in datatype:
763 name = datatype
803 name = datatype
764 else:
804 else:
765 name = '%sReader' %(datatype)
805 name = '%sReader' %(datatype)
766
806
767 if datatype==None:
807 if datatype==None:
768 datatype = name.replace('Reader','')
808 datatype = name.replace('Reader','')
769
809
770 self.datatype = datatype
810 self.datatype = datatype
771 self.name = name
811 self.name = name
772 self.path = path
812 self.path = path
773 self.startDate = startDate
813 self.startDate = startDate
774 self.endDate = endDate
814 self.endDate = endDate
775 self.startTime = startTime
815 self.startTime = startTime
776 self.endTime = endTime
816 self.endTime = endTime
777
817
778 self.inputId = '0'
818 self.inputId = '0'
779 self.parentId = parentId
819 self.parentId = parentId
780
820
781 self.updateRunOperation(**kwargs)
821 self.updateRunOperation(**kwargs)
782
822
783 def removeOperations(self):
823 def removeOperations(self):
784
824
785 for obj in self.opConfObjList:
825 for obj in self.opConfObjList:
786 del obj
826 del obj
787
827
788 self.opConfObjList = []
828 self.opConfObjList = []
789
829
790 def addRunOperation(self, **kwargs):
830 def addRunOperation(self, **kwargs):
791
831
792 opObj = self.addOperation(name = 'run', optype = 'self')
832 opObj = self.addOperation(name = 'run', optype = 'self')
833
834 if self.server is None:
835 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
836 opObj.addParameter(name='path' , value=self.path, format='str')
837 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
838 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
839 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
840 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
841 opObj.addParameter(name='queue' , value=self.queue, format='obj')
842 for key, value in kwargs.items():
843 opObj.addParameter(name=key, value=value, format=type(value).__name__)
844 else:
845 opObj.addParameter(name='server' , value=self.server, format='str')
793
846
794 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
847
795 opObj.addParameter(name='path' , value=self.path, format='str')
796 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
797 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
798 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
799 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
800
801 for key, value in kwargs.items():
802 opObj.addParameter(name=key, value=value, format=type(value).__name__)
803
804 return opObj
848 return opObj
805
849
806 def updateRunOperation(self, **kwargs):
850 def updateRunOperation(self, **kwargs):
807
851
808 opObj = self.getOperationObj(name = 'run')
852 opObj = self.getOperationObj(name = 'run')
809 opObj.removeParameters()
853 opObj.removeParameters()
810
854
811 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
855 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
812 opObj.addParameter(name='path' , value=self.path, format='str')
856 opObj.addParameter(name='path' , value=self.path, format='str')
813 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
857 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
814 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
858 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
815 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
859 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
816 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
860 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
817
861
818 for key, value in kwargs.items():
862 for key, value in kwargs.items():
819 opObj.addParameter(name=key, value=value, format=type(value).__name__)
863 opObj.addParameter(name=key, value=value, format=type(value).__name__)
820
864
821 return opObj
865 return opObj
822
866
823 # def makeXml(self, projectElement):
867 # def makeXml(self, projectElement):
824 #
868 #
825 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
869 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
826 # procUnitElement.set('id', str(self.id))
870 # procUnitElement.set('id', str(self.id))
827 # procUnitElement.set('name', self.name)
871 # procUnitElement.set('name', self.name)
828 # procUnitElement.set('datatype', self.datatype)
872 # procUnitElement.set('datatype', self.datatype)
829 # procUnitElement.set('inputId', str(self.inputId))
873 # procUnitElement.set('inputId', str(self.inputId))
830 #
874 #
831 # for opConfObj in self.opConfObjList:
875 # for opConfObj in self.opConfObjList:
832 # opConfObj.makeXml(procUnitElement)
876 # opConfObj.makeXml(procUnitElement)
833
877
834 def readXml(self, upElement):
878 def readXml(self, upElement):
835
879
836 self.id = upElement.get('id')
880 self.id = upElement.get('id')
837 self.name = upElement.get('name')
881 self.name = upElement.get('name')
838 self.datatype = upElement.get('datatype')
882 self.datatype = upElement.get('datatype')
839 self.inputId = upElement.get('inputId')
883 self.inputId = upElement.get('inputId')
840
884
841 if self.ELEMENTNAME == "ReadUnit":
885 if self.ELEMENTNAME == "ReadUnit":
842 self.datatype = self.datatype.replace("Reader", "")
886 self.datatype = self.datatype.replace("Reader", "")
843
887
844 if self.inputId == 'None':
888 if self.inputId == 'None':
845 self.inputId = '0'
889 self.inputId = '0'
846
890
847 self.opConfObjList = []
891 self.opConfObjList = []
848
892
849 opElementList = upElement.iter(OperationConf().getElementName())
893 opElementList = upElement.iter(OperationConf().getElementName())
850
894
851 for opElement in opElementList:
895 for opElement in opElementList:
852 opConfObj = OperationConf()
896 opConfObj = OperationConf()
853 opConfObj.readXml(opElement)
897 opConfObj.readXml(opElement)
854 self.opConfObjList.append(opConfObj)
898 self.opConfObjList.append(opConfObj)
855
899
856 if opConfObj.name == 'run':
900 if opConfObj.name == 'run':
857 self.path = opConfObj.getParameterValue('path')
901 self.path = opConfObj.getParameterValue('path')
858 self.startDate = opConfObj.getParameterValue('startDate')
902 self.startDate = opConfObj.getParameterValue('startDate')
859 self.endDate = opConfObj.getParameterValue('endDate')
903 self.endDate = opConfObj.getParameterValue('endDate')
860 self.startTime = opConfObj.getParameterValue('startTime')
904 self.startTime = opConfObj.getParameterValue('startTime')
861 self.endTime = opConfObj.getParameterValue('endTime')
905 self.endTime = opConfObj.getParameterValue('endTime')
862
906
863 class Project(Process):
907 class Project(Process):
864 id = None
908 id = None
865 name = None
909 name = None
866 description = None
910 description = None
867 filename = None
911 filename = None
868
912
869 procUnitConfObjDict = None
913 procUnitConfObjDict = None
870
914
871 ELEMENTNAME = 'Project'
915 ELEMENTNAME = 'Project'
872
916
873 plotterQueue = None
917 plotterQueue = None
874
918
875 def __init__(self, plotter_queue=None, logfile=None):
919 def __init__(self, plotter_queue=None, logfile=None):
876 Process.__init__(self)
920 Process.__init__(self)
877 self.id = None
921 self.id = None
878 self.name = None
922 self.name = None
879 self.description = None
923 self.description = None
880 if logfile is not None:
924 if logfile is not None:
881 logToFile(logfile)
925 logToFile(logfile)
882 self.plotterQueue = plotter_queue
926 self.plotterQueue = plotter_queue
883
927
884 self.procUnitConfObjDict = {}
928 self.procUnitConfObjDict = {}
885
929
886 def __getNewId(self):
930 def __getNewId(self):
887
931
888 idList = self.procUnitConfObjDict.keys()
932 idList = self.procUnitConfObjDict.keys()
889
933
890 id = int(self.id)*10
934 id = int(self.id)*10
891
935
892 while True:
936 while True:
893 id += 1
937 id += 1
894
938
895 if str(id) in idList:
939 if str(id) in idList:
896 continue
940 continue
897
941
898 break
942 break
899
943
900 return str(id)
944 return str(id)
901
945
902 def getElementName(self):
946 def getElementName(self):
903
947
904 return self.ELEMENTNAME
948 return self.ELEMENTNAME
905
949
906 def getId(self):
950 def getId(self):
907
951
908 return self.id
952 return self.id
909
953
910 def updateId(self, new_id):
954 def updateId(self, new_id):
911
955
912 self.id = str(new_id)
956 self.id = str(new_id)
913
957
914 keyList = self.procUnitConfObjDict.keys()
958 keyList = self.procUnitConfObjDict.keys()
915 keyList.sort()
959 keyList.sort()
916
960
917 n = 1
961 n = 1
918 newProcUnitConfObjDict = {}
962 newProcUnitConfObjDict = {}
919
963
920 for procKey in keyList:
964 for procKey in keyList:
921
965
922 procUnitConfObj = self.procUnitConfObjDict[procKey]
966 procUnitConfObj = self.procUnitConfObjDict[procKey]
923 idProcUnit = str(int(self.id)*10 + n)
967 idProcUnit = str(int(self.id)*10 + n)
924 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
968 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
925
969
926 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
970 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
927 n += 1
971 n += 1
928
972
929 self.procUnitConfObjDict = newProcUnitConfObjDict
973 self.procUnitConfObjDict = newProcUnitConfObjDict
930
974
931 def setup(self, id, name, description):
975 def setup(self, id, name, description):
932
976
933 self.id = str(id)
977 self.id = str(id)
934 self.name = name
978 self.name = name
935 self.description = description
979 self.description = description
936
980
937 def update(self, name, description):
981 def update(self, name, description):
938
982
939 self.name = name
983 self.name = name
940 self.description = description
984 self.description = description
941
985
942 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
986 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
943 if id is None:
987 if id is None:
944 idReadUnit = self.__getNewId()
988 idReadUnit = self.__getNewId()
945 else:
989 else:
946 idReadUnit = str(id)
990 idReadUnit = str(id)
947
991
948 readUnitConfObj = ReadUnitConf()
992 readUnitConfObj = ReadUnitConf()
949 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
993 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
950
994
951 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
995 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
952
996
953 return readUnitConfObj
997 return readUnitConfObj
954
998
955 def addProcUnit(self, inputId='0', datatype=None, name=None):
999 def addProcUnit(self, inputId='0', datatype=None, name=None):
956
1000
957 idProcUnit = self.__getNewId()
1001 idProcUnit = self.__getNewId()
958
1002
959 procUnitConfObj = ProcUnitConf()
1003 procUnitConfObj = ProcUnitConf()
960 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1004 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
961
1005
962 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1006 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
963
1007
964 return procUnitConfObj
1008 return procUnitConfObj
965
1009
966 def removeProcUnit(self, id):
1010 def removeProcUnit(self, id):
967
1011
968 if id in self.procUnitConfObjDict.keys():
1012 if id in self.procUnitConfObjDict.keys():
969 self.procUnitConfObjDict.pop(id)
1013 self.procUnitConfObjDict.pop(id)
970
1014
971 def getReadUnitId(self):
1015 def getReadUnitId(self):
972
1016
973 readUnitConfObj = self.getReadUnitObj()
1017 readUnitConfObj = self.getReadUnitObj()
974
1018
975 return readUnitConfObj.id
1019 return readUnitConfObj.id
976
1020
977 def getReadUnitObj(self):
1021 def getReadUnitObj(self):
978
1022
979 for obj in self.procUnitConfObjDict.values():
1023 for obj in self.procUnitConfObjDict.values():
980 if obj.getElementName() == "ReadUnit":
1024 if obj.getElementName() == "ReadUnit":
981 return obj
1025 return obj
982
1026
983 return None
1027 return None
984
1028
985 def getProcUnitObj(self, id=None, name=None):
1029 def getProcUnitObj(self, id=None, name=None):
986
1030
987 if id != None:
1031 if id != None:
988 return self.procUnitConfObjDict[id]
1032 return self.procUnitConfObjDict[id]
989
1033
990 if name != None:
1034 if name != None:
991 return self.getProcUnitObjByName(name)
1035 return self.getProcUnitObjByName(name)
992
1036
993 return None
1037 return None
994
1038
995 def getProcUnitObjByName(self, name):
1039 def getProcUnitObjByName(self, name):
996
1040
997 for obj in self.procUnitConfObjDict.values():
1041 for obj in self.procUnitConfObjDict.values():
998 if obj.name == name:
1042 if obj.name == name:
999 return obj
1043 return obj
1000
1044
1001 return None
1045 return None
1002
1046
1003 def procUnitItems(self):
1047 def procUnitItems(self):
1004
1048
1005 return self.procUnitConfObjDict.items()
1049 return self.procUnitConfObjDict.items()
1006
1050
1007 def makeXml(self):
1051 def makeXml(self):
1008
1052
1009 projectElement = Element('Project')
1053 projectElement = Element('Project')
1010 projectElement.set('id', str(self.id))
1054 projectElement.set('id', str(self.id))
1011 projectElement.set('name', self.name)
1055 projectElement.set('name', self.name)
1012 projectElement.set('description', self.description)
1056 projectElement.set('description', self.description)
1013
1057
1014 for procUnitConfObj in self.procUnitConfObjDict.values():
1058 for procUnitConfObj in self.procUnitConfObjDict.values():
1015 procUnitConfObj.makeXml(projectElement)
1059 procUnitConfObj.makeXml(projectElement)
1016
1060
1017 self.projectElement = projectElement
1061 self.projectElement = projectElement
1018
1062
1019 def writeXml(self, filename=None):
1063 def writeXml(self, filename=None):
1020
1064
1021 if filename == None:
1065 if filename == None:
1022 if self.filename:
1066 if self.filename:
1023 filename = self.filename
1067 filename = self.filename
1024 else:
1068 else:
1025 filename = "schain.xml"
1069 filename = "schain.xml"
1026
1070
1027 if not filename:
1071 if not filename:
1028 print "filename has not been defined. Use setFilename(filename) for do it."
1072 print "filename has not been defined. Use setFilename(filename) for do it."
1029 return 0
1073 return 0
1030
1074
1031 abs_file = os.path.abspath(filename)
1075 abs_file = os.path.abspath(filename)
1032
1076
1033 if not os.access(os.path.dirname(abs_file), os.W_OK):
1077 if not os.access(os.path.dirname(abs_file), os.W_OK):
1034 print "No write permission on %s" %os.path.dirname(abs_file)
1078 print "No write permission on %s" %os.path.dirname(abs_file)
1035 return 0
1079 return 0
1036
1080
1037 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1081 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1038 print "File %s already exists and it could not be overwriten" %abs_file
1082 print "File %s already exists and it could not be overwriten" %abs_file
1039 return 0
1083 return 0
1040
1084
1041 self.makeXml()
1085 self.makeXml()
1042
1086
1043 ElementTree(self.projectElement).write(abs_file, method='xml')
1087 ElementTree(self.projectElement).write(abs_file, method='xml')
1044
1088
1045 self.filename = abs_file
1089 self.filename = abs_file
1046
1090
1047 return 1
1091 return 1
1048
1092
1049 def readXml(self, filename = None):
1093 def readXml(self, filename = None):
1050
1094
1051 if not filename:
1095 if not filename:
1052 print "filename is not defined"
1096 print "filename is not defined"
1053 return 0
1097 return 0
1054
1098
1055 abs_file = os.path.abspath(filename)
1099 abs_file = os.path.abspath(filename)
1056
1100
1057 if not os.path.isfile(abs_file):
1101 if not os.path.isfile(abs_file):
1058 print "%s file does not exist" %abs_file
1102 print "%s file does not exist" %abs_file
1059 return 0
1103 return 0
1060
1104
1061 self.projectElement = None
1105 self.projectElement = None
1062 self.procUnitConfObjDict = {}
1106 self.procUnitConfObjDict = {}
1063
1107
1064 try:
1108 try:
1065 self.projectElement = ElementTree().parse(abs_file)
1109 self.projectElement = ElementTree().parse(abs_file)
1066 except:
1110 except:
1067 print "Error reading %s, verify file format" %filename
1111 print "Error reading %s, verify file format" %filename
1068 return 0
1112 return 0
1069
1113
1070 self.project = self.projectElement.tag
1114 self.project = self.projectElement.tag
1071
1115
1072 self.id = self.projectElement.get('id')
1116 self.id = self.projectElement.get('id')
1073 self.name = self.projectElement.get('name')
1117 self.name = self.projectElement.get('name')
1074 self.description = self.projectElement.get('description')
1118 self.description = self.projectElement.get('description')
1075
1119
1076 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1120 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1077
1121
1078 for readUnitElement in readUnitElementList:
1122 for readUnitElement in readUnitElementList:
1079 readUnitConfObj = ReadUnitConf()
1123 readUnitConfObj = ReadUnitConf()
1080 readUnitConfObj.readXml(readUnitElement)
1124 readUnitConfObj.readXml(readUnitElement)
1081
1125
1082 if readUnitConfObj.parentId == None:
1126 if readUnitConfObj.parentId == None:
1083 readUnitConfObj.parentId = self.id
1127 readUnitConfObj.parentId = self.id
1084
1128
1085 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1129 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1086
1130
1087 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1131 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1088
1132
1089 for procUnitElement in procUnitElementList:
1133 for procUnitElement in procUnitElementList:
1090 procUnitConfObj = ProcUnitConf()
1134 procUnitConfObj = ProcUnitConf()
1091 procUnitConfObj.readXml(procUnitElement)
1135 procUnitConfObj.readXml(procUnitElement)
1092
1136
1093 if procUnitConfObj.parentId == None:
1137 if procUnitConfObj.parentId == None:
1094 procUnitConfObj.parentId = self.id
1138 procUnitConfObj.parentId = self.id
1095
1139
1096 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1140 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1097
1141
1098 self.filename = abs_file
1142 self.filename = abs_file
1099
1143
1100 return 1
1144 return 1
1101
1145
1102 def printattr(self):
1146 def printattr(self):
1103
1147
1104 print "Project[%s]: name = %s, description = %s" %(self.id,
1148 print "Project[%s]: name = %s, description = %s" %(self.id,
1105 self.name,
1149 self.name,
1106 self.description)
1150 self.description)
1107
1151
1108 for procUnitConfObj in self.procUnitConfObjDict.values():
1152 for procUnitConfObj in self.procUnitConfObjDict.values():
1109 procUnitConfObj.printattr()
1153 procUnitConfObj.printattr()
1110
1154
1111 def createObjects(self):
1155 def createObjects(self):
1112
1156
1113 for procUnitConfObj in self.procUnitConfObjDict.values():
1157 for procUnitConfObj in self.procUnitConfObjDict.values():
1114 procUnitConfObj.createObjects(self.plotterQueue)
1158 procUnitConfObj.createObjects(self.plotterQueue)
1115
1159
1116 def __connect(self, objIN, thisObj):
1160 def __connect(self, objIN, thisObj):
1117
1161
1118 thisObj.setInput(objIN.getOutputObj())
1162 thisObj.setInput(objIN.getOutputObj())
1119
1163
1120 def connectObjects(self):
1164 def connectObjects(self):
1121
1165
1122 for thisPUConfObj in self.procUnitConfObjDict.values():
1166 for thisPUConfObj in self.procUnitConfObjDict.values():
1123
1167
1124 inputId = thisPUConfObj.getInputId()
1168 inputId = thisPUConfObj.getInputId()
1125
1169
1126 if int(inputId) == 0:
1170 if int(inputId) == 0:
1127 continue
1171 continue
1128
1172
1129 #Get input object
1173 #Get input object
1130 puConfINObj = self.procUnitConfObjDict[inputId]
1174 puConfINObj = self.procUnitConfObjDict[inputId]
1131 puObjIN = puConfINObj.getProcUnitObj()
1175 puObjIN = puConfINObj.getProcUnitObj()
1132
1176
1133 #Get current object
1177 #Get current object
1134 thisPUObj = thisPUConfObj.getProcUnitObj()
1178 thisPUObj = thisPUConfObj.getProcUnitObj()
1135
1179
1136 self.__connect(puObjIN, thisPUObj)
1180 self.__connect(puObjIN, thisPUObj)
1137
1181
1138 def __handleError(self, procUnitConfObj, send_email=True):
1182 def __handleError(self, procUnitConfObj, send_email=True):
1139
1183
1140 import socket
1184 import socket
1141
1185
1142 err = traceback.format_exception(sys.exc_info()[0],
1186 err = traceback.format_exception(sys.exc_info()[0],
1143 sys.exc_info()[1],
1187 sys.exc_info()[1],
1144 sys.exc_info()[2])
1188 sys.exc_info()[2])
1145
1189
1146 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1190 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1147 print "***** %s" %err[-1]
1191 print "***** %s" %err[-1]
1148
1192
1149 message = "".join(err)
1193 message = "".join(err)
1150
1194
1151 sys.stderr.write(message)
1195 sys.stderr.write(message)
1152
1196
1153 if not send_email:
1197 if not send_email:
1154 return
1198 return
1155
1199
1156 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1200 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1157
1201
1158 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1202 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1159 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1203 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1160 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1204 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1161 subtitle += "Configuration file: %s\n" %self.filename
1205 subtitle += "Configuration file: %s\n" %self.filename
1162 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1206 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1163
1207
1164 readUnitConfObj = self.getReadUnitObj()
1208 readUnitConfObj = self.getReadUnitObj()
1165 if readUnitConfObj:
1209 if readUnitConfObj:
1166 subtitle += "\nInput parameters:\n"
1210 subtitle += "\nInput parameters:\n"
1167 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1211 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1168 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1212 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1169 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1213 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1170 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1214 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1171 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1215 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1172 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1216 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1173
1217
1174 adminObj = schainpy.admin.SchainNotify()
1218 adminObj = schainpy.admin.SchainNotify()
1175 adminObj.sendAlert(message=message,
1219 adminObj.sendAlert(message=message,
1176 subject=subject,
1220 subject=subject,
1177 subtitle=subtitle,
1221 subtitle=subtitle,
1178 filename=self.filename)
1222 filename=self.filename)
1179
1223
1180 def isPaused(self):
1224 def isPaused(self):
1181 return 0
1225 return 0
1182
1226
1183 def isStopped(self):
1227 def isStopped(self):
1184 return 0
1228 return 0
1185
1229
1186 def runController(self):
1230 def runController(self):
1187 """
1231 """
1188 returns 0 when this process has been stopped, 1 otherwise
1232 returns 0 when this process has been stopped, 1 otherwise
1189 """
1233 """
1190
1234
1191 if self.isPaused():
1235 if self.isPaused():
1192 print "Process suspended"
1236 print "Process suspended"
1193
1237
1194 while True:
1238 while True:
1195 sleep(0.1)
1239 sleep(0.1)
1196
1240
1197 if not self.isPaused():
1241 if not self.isPaused():
1198 break
1242 break
1199
1243
1200 if self.isStopped():
1244 if self.isStopped():
1201 break
1245 break
1202
1246
1203 print "Process reinitialized"
1247 print "Process reinitialized"
1204
1248
1205 if self.isStopped():
1249 if self.isStopped():
1206 print "Process stopped"
1250 print "Process stopped"
1207 return 0
1251 return 0
1208
1252
1209 return 1
1253 return 1
1210
1254
1211 def setFilename(self, filename):
1255 def setFilename(self, filename):
1212
1256
1213 self.filename = filename
1257 self.filename = filename
1214
1258
1215 def setPlotterQueue(self, plotter_queue):
1259 def setPlotterQueue(self, plotter_queue):
1216
1260
1217 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1261 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1218
1262
1219 def getPlotterQueue(self):
1263 def getPlotterQueue(self):
1220
1264
1221 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1222
1266
1223 def useExternalPlotter(self):
1267 def useExternalPlotter(self):
1224
1268
1225 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1226
1270
1227
1271
1228 def run(self, filename=None):
1272 def run(self, filename=None):
1229
1273
1230 # self.writeXml(filename)
1274 # self.writeXml(filename)
1231 self.createObjects()
1275 self.createObjects()
1232 self.connectObjects()
1276 self.connectObjects()
1233
1277
1234 print
1278 print
1235 print "*"*60
1279 print "*"*60
1236 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1280 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1237 print "*"*60
1281 print "*"*60
1238 print
1282 print
1239
1283
1240 keyList = self.procUnitConfObjDict.keys()
1284 keyList = self.procUnitConfObjDict.keys()
1241 keyList.sort()
1285 keyList.sort()
1242
1286
1243 while(True):
1287 while(True):
1244
1288
1245 is_ok = False
1289 is_ok = False
1246
1290
1247 for procKey in keyList:
1291 for procKey in keyList:
1248 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1292 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1249
1293
1250 procUnitConfObj = self.procUnitConfObjDict[procKey]
1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1251
1295
1252 try:
1296 try:
1253 sts = procUnitConfObj.run()
1297 sts = procUnitConfObj.run()
1254 is_ok = is_ok or sts
1298 is_ok = is_ok or sts
1255 except KeyboardInterrupt:
1299 except KeyboardInterrupt:
1256 is_ok = False
1300 is_ok = False
1257 break
1301 break
1258 except ValueError, e:
1302 except ValueError, e:
1259 sleep(0.5)
1303 sleep(0.5)
1260 self.__handleError(procUnitConfObj, send_email=True)
1304 self.__handleError(procUnitConfObj, send_email=True)
1261 is_ok = False
1305 is_ok = False
1262 break
1306 break
1263 except:
1307 except:
1264 sleep(0.5)
1308 sleep(0.5)
1265 self.__handleError(procUnitConfObj)
1309 self.__handleError(procUnitConfObj)
1266 is_ok = False
1310 is_ok = False
1267 break
1311 break
1268
1312
1269 #If every process unit finished so end process
1313 #If every process unit finished so end process
1270 if not(is_ok):
1314 if not(is_ok):
1271 # print "Every process unit have finished"
1315 # print "Every process unit have finished"
1272 break
1316 break
1273
1317
1274 if not self.runController():
1318 if not self.runController():
1275 break
1319 break
1276
1320
1277 #Closing every process
1321 #Closing every process
1278 for procKey in keyList:
1322 for procKey in keyList:
1279 procUnitConfObj = self.procUnitConfObjDict[procKey]
1323 procUnitConfObj = self.procUnitConfObjDict[procKey]
1280 procUnitConfObj.close()
1324 procUnitConfObj.close()
General Comments 0
You need to be logged in to leave comments. Login now