##// END OF EJS Templates
schain
RSS
Hotfix in controller.Project.update()
jespinoza -
r1133:682617c0adc9
parent child
Show More
Side by Side Unified
Context file:
r1133:682617c0adc9 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1330 +1,1330
1 '''
1 '''
2 Created on September , 2012
2 Created on September , 2012
3 @author:
3 @author:
4 '''
4 '''
5
5
6 import sys
6 import sys
7 import ast
7 import ast
8 import datetime
8 import datetime
9 import traceback
9 import traceback
10 import math
10 import math
11 import time
11 import time
12 from multiprocessing import Process, cpu_count
12 from multiprocessing import Process, cpu_count
13
13
14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
15 from xml.dom import minidom
15 from xml.dom import minidom
16
16
17 import schainpy
17 import schainpy
18 from schainpy.admin import Alarm, SchainWarning
18 from schainpy.admin import Alarm, SchainWarning
19 from schainpy.model import *
19 from schainpy.model import *
20 from schainpy.utils import log
20 from schainpy.utils import log
21
21
22 DTYPES = {
22 DTYPES = {
23 'Voltage': '.r',
23 'Voltage': '.r',
24 'Spectra': '.pdata'
24 'Spectra': '.pdata'
25 }
25 }
26
26
27
27
28 def MPProject(project, n=cpu_count()):
28 def MPProject(project, n=cpu_count()):
29 '''
29 '''
30 Project wrapper to run schain in n processes
30 Project wrapper to run schain in n processes
31 '''
31 '''
32
32
33 rconf = project.getReadUnitObj()
33 rconf = project.getReadUnitObj()
34 op = rconf.getOperationObj('run')
34 op = rconf.getOperationObj('run')
35 dt1 = op.getParameterValue('startDate')
35 dt1 = op.getParameterValue('startDate')
36 dt2 = op.getParameterValue('endDate')
36 dt2 = op.getParameterValue('endDate')
37 tm1 = op.getParameterValue('startTime')
37 tm1 = op.getParameterValue('startTime')
38 tm2 = op.getParameterValue('endTime')
38 tm2 = op.getParameterValue('endTime')
39 days = (dt2 - dt1).days
39 days = (dt2 - dt1).days
40
40
41 for day in range(days + 1):
41 for day in range(days + 1):
42 skip = 0
42 skip = 0
43 cursor = 0
43 cursor = 0
44 processes = []
44 processes = []
45 dt = dt1 + datetime.timedelta(day)
45 dt = dt1 + datetime.timedelta(day)
46 dt_str = dt.strftime('%Y/%m/%d')
46 dt_str = dt.strftime('%Y/%m/%d')
47 reader = JRODataReader()
47 reader = JRODataReader()
48 paths, files = reader.searchFilesOffLine(path=rconf.path,
48 paths, files = reader.searchFilesOffLine(path=rconf.path,
49 startDate=dt,
49 startDate=dt,
50 endDate=dt,
50 endDate=dt,
51 startTime=tm1,
51 startTime=tm1,
52 endTime=tm2,
52 endTime=tm2,
53 ext=DTYPES[rconf.datatype])
53 ext=DTYPES[rconf.datatype])
54 nFiles = len(files)
54 nFiles = len(files)
55 if nFiles == 0:
55 if nFiles == 0:
56 continue
56 continue
57 skip = int(math.ceil(nFiles / n))
57 skip = int(math.ceil(nFiles / n))
58 while nFiles > cursor * skip:
58 while nFiles > cursor * skip:
59 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
59 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
60 skip=skip)
60 skip=skip)
61 p = project.clone()
61 p = project.clone()
62 p.start()
62 p.start()
63 processes.append(p)
63 processes.append(p)
64 cursor += 1
64 cursor += 1
65
65
66 def beforeExit(exctype, value, trace):
66 def beforeExit(exctype, value, trace):
67 for process in processes:
67 for process in processes:
68 process.terminate()
68 process.terminate()
69 process.join()
69 process.join()
70 print traceback.print_tb(trace)
70 print traceback.print_tb(trace)
71
71
72 sys.excepthook = beforeExit
72 sys.excepthook = beforeExit
73
73
74 for process in processes:
74 for process in processes:
75 process.join()
75 process.join()
76 process.terminate()
76 process.terminate()
77
77
78 time.sleep(3)
78 time.sleep(3)
79
79
80
80
81 class ParameterConf():
81 class ParameterConf():
82
82
83 id = None
83 id = None
84 name = None
84 name = None
85 value = None
85 value = None
86 format = None
86 format = None
87
87
88 __formated_value = None
88 __formated_value = None
89
89
90 ELEMENTNAME = 'Parameter'
90 ELEMENTNAME = 'Parameter'
91
91
92 def __init__(self):
92 def __init__(self):
93
93
94 self.format = 'str'
94 self.format = 'str'
95
95
96 def getElementName(self):
96 def getElementName(self):
97
97
98 return self.ELEMENTNAME
98 return self.ELEMENTNAME
99
99
100 def getValue(self):
100 def getValue(self):
101
101
102 value = self.value
102 value = self.value
103 format = self.format
103 format = self.format
104
104
105 if self.__formated_value != None:
105 if self.__formated_value != None:
106
106
107 return self.__formated_value
107 return self.__formated_value
108
108
109 if format == 'obj':
109 if format == 'obj':
110 return value
110 return value
111
111
112 if format == 'str':
112 if format == 'str':
113 self.__formated_value = str(value)
113 self.__formated_value = str(value)
114 return self.__formated_value
114 return self.__formated_value
115
115
116 if value == '':
116 if value == '':
117 raise ValueError, '%s: This parameter value is empty' % self.name
117 raise ValueError, '%s: This parameter value is empty' % self.name
118
118
119 if format == 'list':
119 if format == 'list':
120 strList = value.split(',')
120 strList = value.split(',')
121
121
122 self.__formated_value = strList
122 self.__formated_value = strList
123
123
124 return self.__formated_value
124 return self.__formated_value
125
125
126 if format == 'intlist':
126 if format == 'intlist':
127 '''
127 '''
128 Example:
128 Example:
129 value = (0,1,2)
129 value = (0,1,2)
130 '''
130 '''
131
131
132 new_value = ast.literal_eval(value)
132 new_value = ast.literal_eval(value)
133
133
134 if type(new_value) not in (tuple, list):
134 if type(new_value) not in (tuple, list):
135 new_value = [int(new_value)]
135 new_value = [int(new_value)]
136
136
137 self.__formated_value = new_value
137 self.__formated_value = new_value
138
138
139 return self.__formated_value
139 return self.__formated_value
140
140
141 if format == 'floatlist':
141 if format == 'floatlist':
142 '''
142 '''
143 Example:
143 Example:
144 value = (0.5, 1.4, 2.7)
144 value = (0.5, 1.4, 2.7)
145 '''
145 '''
146
146
147 new_value = ast.literal_eval(value)
147 new_value = ast.literal_eval(value)
148
148
149 if type(new_value) not in (tuple, list):
149 if type(new_value) not in (tuple, list):
150 new_value = [float(new_value)]
150 new_value = [float(new_value)]
151
151
152 self.__formated_value = new_value
152 self.__formated_value = new_value
153
153
154 return self.__formated_value
154 return self.__formated_value
155
155
156 if format == 'date':
156 if format == 'date':
157 strList = value.split('/')
157 strList = value.split('/')
158 intList = [int(x) for x in strList]
158 intList = [int(x) for x in strList]
159 date = datetime.date(intList[0], intList[1], intList[2])
159 date = datetime.date(intList[0], intList[1], intList[2])
160
160
161 self.__formated_value = date
161 self.__formated_value = date
162
162
163 return self.__formated_value
163 return self.__formated_value
164
164
165 if format == 'time':
165 if format == 'time':
166 strList = value.split(':')
166 strList = value.split(':')
167 intList = [int(x) for x in strList]
167 intList = [int(x) for x in strList]
168 time = datetime.time(intList[0], intList[1], intList[2])
168 time = datetime.time(intList[0], intList[1], intList[2])
169
169
170 self.__formated_value = time
170 self.__formated_value = time
171
171
172 return self.__formated_value
172 return self.__formated_value
173
173
174 if format == 'pairslist':
174 if format == 'pairslist':
175 '''
175 '''
176 Example:
176 Example:
177 value = (0,1),(1,2)
177 value = (0,1),(1,2)
178 '''
178 '''
179
179
180 new_value = ast.literal_eval(value)
180 new_value = ast.literal_eval(value)
181
181
182 if type(new_value) not in (tuple, list):
182 if type(new_value) not in (tuple, list):
183 raise ValueError, '%s has to be a tuple or list of pairs' % value
183 raise ValueError, '%s has to be a tuple or list of pairs' % value
184
184
185 if type(new_value[0]) not in (tuple, list):
185 if type(new_value[0]) not in (tuple, list):
186 if len(new_value) != 2:
186 if len(new_value) != 2:
187 raise ValueError, '%s has to be a tuple or list of pairs' % value
187 raise ValueError, '%s has to be a tuple or list of pairs' % value
188 new_value = [new_value]
188 new_value = [new_value]
189
189
190 for thisPair in new_value:
190 for thisPair in new_value:
191 if len(thisPair) != 2:
191 if len(thisPair) != 2:
192 raise ValueError, '%s has to be a tuple or list of pairs' % value
192 raise ValueError, '%s has to be a tuple or list of pairs' % value
193
193
194 self.__formated_value = new_value
194 self.__formated_value = new_value
195
195
196 return self.__formated_value
196 return self.__formated_value
197
197
198 if format == 'multilist':
198 if format == 'multilist':
199 '''
199 '''
200 Example:
200 Example:
201 value = (0,1,2),(3,4,5)
201 value = (0,1,2),(3,4,5)
202 '''
202 '''
203 multiList = ast.literal_eval(value)
203 multiList = ast.literal_eval(value)
204
204
205 if type(multiList[0]) == int:
205 if type(multiList[0]) == int:
206 multiList = ast.literal_eval('(' + value + ')')
206 multiList = ast.literal_eval('(' + value + ')')
207
207
208 self.__formated_value = multiList
208 self.__formated_value = multiList
209
209
210 return self.__formated_value
210 return self.__formated_value
211
211
212 if format == 'bool':
212 if format == 'bool':
213 value = int(value)
213 value = int(value)
214
214
215 if format == 'int':
215 if format == 'int':
216 value = float(value)
216 value = float(value)
217
217
218 format_func = eval(format)
218 format_func = eval(format)
219
219
220 self.__formated_value = format_func(value)
220 self.__formated_value = format_func(value)
221
221
222 return self.__formated_value
222 return self.__formated_value
223
223
224 def updateId(self, new_id):
224 def updateId(self, new_id):
225
225
226 self.id = str(new_id)
226 self.id = str(new_id)
227
227
228 def setup(self, id, name, value, format='str'):
228 def setup(self, id, name, value, format='str'):
229 self.id = str(id)
229 self.id = str(id)
230 self.name = name
230 self.name = name
231 if format == 'obj':
231 if format == 'obj':
232 self.value = value
232 self.value = value
233 else:
233 else:
234 self.value = str(value)
234 self.value = str(value)
235 self.format = str.lower(format)
235 self.format = str.lower(format)
236
236
237 self.getValue()
237 self.getValue()
238
238
239 return 1
239 return 1
240
240
241 def update(self, name, value, format='str'):
241 def update(self, name, value, format='str'):
242
242
243 self.name = name
243 self.name = name
244 self.value = str(value)
244 self.value = str(value)
245 self.format = format
245 self.format = format
246
246
247 def makeXml(self, opElement):
247 def makeXml(self, opElement):
248 if self.name not in ('queue',):
248 if self.name not in ('queue',):
249 parmElement = SubElement(opElement, self.ELEMENTNAME)
249 parmElement = SubElement(opElement, self.ELEMENTNAME)
250 parmElement.set('id', str(self.id))
250 parmElement.set('id', str(self.id))
251 parmElement.set('name', self.name)
251 parmElement.set('name', self.name)
252 parmElement.set('value', self.value)
252 parmElement.set('value', self.value)
253 parmElement.set('format', self.format)
253 parmElement.set('format', self.format)
254
254
255 def readXml(self, parmElement):
255 def readXml(self, parmElement):
256
256
257 self.id = parmElement.get('id')
257 self.id = parmElement.get('id')
258 self.name = parmElement.get('name')
258 self.name = parmElement.get('name')
259 self.value = parmElement.get('value')
259 self.value = parmElement.get('value')
260 self.format = str.lower(parmElement.get('format'))
260 self.format = str.lower(parmElement.get('format'))
261
261
262 # Compatible with old signal chain version
262 # Compatible with old signal chain version
263 if self.format == 'int' and self.name == 'idfigure':
263 if self.format == 'int' and self.name == 'idfigure':
264 self.name = 'id'
264 self.name = 'id'
265
265
266 def printattr(self):
266 def printattr(self):
267
267
268 print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
268 print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
269
269
270
270
271 class OperationConf():
271 class OperationConf():
272
272
273 id = None
273 id = None
274 name = None
274 name = None
275 priority = None
275 priority = None
276 type = None
276 type = None
277
277
278 parmConfObjList = []
278 parmConfObjList = []
279
279
280 ELEMENTNAME = 'Operation'
280 ELEMENTNAME = 'Operation'
281
281
282 def __init__(self):
282 def __init__(self):
283
283
284 self.id = '0'
284 self.id = '0'
285 self.name = None
285 self.name = None
286 self.priority = None
286 self.priority = None
287 self.type = 'self'
287 self.type = 'self'
288
288
289 def __getNewId(self):
289 def __getNewId(self):
290
290
291 return int(self.id) * 10 + len(self.parmConfObjList) + 1
291 return int(self.id) * 10 + len(self.parmConfObjList) + 1
292
292
293 def updateId(self, new_id):
293 def updateId(self, new_id):
294
294
295 self.id = str(new_id)
295 self.id = str(new_id)
296
296
297 n = 1
297 n = 1
298 for parmObj in self.parmConfObjList:
298 for parmObj in self.parmConfObjList:
299
299
300 idParm = str(int(new_id) * 10 + n)
300 idParm = str(int(new_id) * 10 + n)
301 parmObj.updateId(idParm)
301 parmObj.updateId(idParm)
302
302
303 n += 1
303 n += 1
304
304
305 def getElementName(self):
305 def getElementName(self):
306
306
307 return self.ELEMENTNAME
307 return self.ELEMENTNAME
308
308
309 def getParameterObjList(self):
309 def getParameterObjList(self):
310
310
311 return self.parmConfObjList
311 return self.parmConfObjList
312
312
313 def getParameterObj(self, parameterName):
313 def getParameterObj(self, parameterName):
314
314
315 for parmConfObj in self.parmConfObjList:
315 for parmConfObj in self.parmConfObjList:
316
316
317 if parmConfObj.name != parameterName:
317 if parmConfObj.name != parameterName:
318 continue
318 continue
319
319
320 return parmConfObj
320 return parmConfObj
321
321
322 return None
322 return None
323
323
324 def getParameterObjfromValue(self, parameterValue):
324 def getParameterObjfromValue(self, parameterValue):
325
325
326 for parmConfObj in self.parmConfObjList:
326 for parmConfObj in self.parmConfObjList:
327
327
328 if parmConfObj.getValue() != parameterValue:
328 if parmConfObj.getValue() != parameterValue:
329 continue
329 continue
330
330
331 return parmConfObj.getValue()
331 return parmConfObj.getValue()
332
332
333 return None
333 return None
334
334
335 def getParameterValue(self, parameterName):
335 def getParameterValue(self, parameterName):
336
336
337 parameterObj = self.getParameterObj(parameterName)
337 parameterObj = self.getParameterObj(parameterName)
338
338
339 # if not parameterObj:
339 # if not parameterObj:
340 # return None
340 # return None
341
341
342 value = parameterObj.getValue()
342 value = parameterObj.getValue()
343
343
344 return value
344 return value
345
345
346 def getKwargs(self):
346 def getKwargs(self):
347
347
348 kwargs = {}
348 kwargs = {}
349
349
350 for parmConfObj in self.parmConfObjList:
350 for parmConfObj in self.parmConfObjList:
351 if self.name == 'run' and parmConfObj.name == 'datatype':
351 if self.name == 'run' and parmConfObj.name == 'datatype':
352 continue
352 continue
353
353
354 kwargs[parmConfObj.name] = parmConfObj.getValue()
354 kwargs[parmConfObj.name] = parmConfObj.getValue()
355
355
356 return kwargs
356 return kwargs
357
357
358 def setup(self, id, name, priority, type):
358 def setup(self, id, name, priority, type):
359
359
360 self.id = str(id)
360 self.id = str(id)
361 self.name = name
361 self.name = name
362 self.type = type
362 self.type = type
363 self.priority = priority
363 self.priority = priority
364
364
365 self.parmConfObjList = []
365 self.parmConfObjList = []
366
366
367 def removeParameters(self):
367 def removeParameters(self):
368
368
369 for obj in self.parmConfObjList:
369 for obj in self.parmConfObjList:
370 del obj
370 del obj
371
371
372 self.parmConfObjList = []
372 self.parmConfObjList = []
373
373
374 def addParameter(self, name, value, format='str'):
374 def addParameter(self, name, value, format='str'):
375
375
376 if value is None:
376 if value is None:
377 return None
377 return None
378 id = self.__getNewId()
378 id = self.__getNewId()
379
379
380 parmConfObj = ParameterConf()
380 parmConfObj = ParameterConf()
381 if not parmConfObj.setup(id, name, value, format):
381 if not parmConfObj.setup(id, name, value, format):
382 return None
382 return None
383
383
384 self.parmConfObjList.append(parmConfObj)
384 self.parmConfObjList.append(parmConfObj)
385
385
386 return parmConfObj
386 return parmConfObj
387
387
388 def changeParameter(self, name, value, format='str'):
388 def changeParameter(self, name, value, format='str'):
389
389
390 parmConfObj = self.getParameterObj(name)
390 parmConfObj = self.getParameterObj(name)
391 parmConfObj.update(name, value, format)
391 parmConfObj.update(name, value, format)
392
392
393 return parmConfObj
393 return parmConfObj
394
394
395 def makeXml(self, procUnitElement):
395 def makeXml(self, procUnitElement):
396
396
397 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
397 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
398 opElement.set('id', str(self.id))
398 opElement.set('id', str(self.id))
399 opElement.set('name', self.name)
399 opElement.set('name', self.name)
400 opElement.set('type', self.type)
400 opElement.set('type', self.type)
401 opElement.set('priority', str(self.priority))
401 opElement.set('priority', str(self.priority))
402
402
403 for parmConfObj in self.parmConfObjList:
403 for parmConfObj in self.parmConfObjList:
404 parmConfObj.makeXml(opElement)
404 parmConfObj.makeXml(opElement)
405
405
406 def readXml(self, opElement):
406 def readXml(self, opElement):
407
407
408 self.id = opElement.get('id')
408 self.id = opElement.get('id')
409 self.name = opElement.get('name')
409 self.name = opElement.get('name')
410 self.type = opElement.get('type')
410 self.type = opElement.get('type')
411 self.priority = opElement.get('priority')
411 self.priority = opElement.get('priority')
412
412
413 # Compatible with old signal chain version
413 # Compatible with old signal chain version
414 # Use of 'run' method instead 'init'
414 # Use of 'run' method instead 'init'
415 if self.type == 'self' and self.name == 'init':
415 if self.type == 'self' and self.name == 'init':
416 self.name = 'run'
416 self.name = 'run'
417
417
418 self.parmConfObjList = []
418 self.parmConfObjList = []
419
419
420 parmElementList = opElement.iter(ParameterConf().getElementName())
420 parmElementList = opElement.iter(ParameterConf().getElementName())
421
421
422 for parmElement in parmElementList:
422 for parmElement in parmElementList:
423 parmConfObj = ParameterConf()
423 parmConfObj = ParameterConf()
424 parmConfObj.readXml(parmElement)
424 parmConfObj.readXml(parmElement)
425
425
426 # Compatible with old signal chain version
426 # Compatible with old signal chain version
427 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
427 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
428 if self.type != 'self' and self.name == 'Plot':
428 if self.type != 'self' and self.name == 'Plot':
429 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
429 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
430 self.name = parmConfObj.value
430 self.name = parmConfObj.value
431 continue
431 continue
432
432
433 self.parmConfObjList.append(parmConfObj)
433 self.parmConfObjList.append(parmConfObj)
434
434
435 def printattr(self):
435 def printattr(self):
436
436
437 print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
437 print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
438 self.id,
438 self.id,
439 self.name,
439 self.name,
440 self.type,
440 self.type,
441 self.priority)
441 self.priority)
442
442
443 for parmConfObj in self.parmConfObjList:
443 for parmConfObj in self.parmConfObjList:
444 parmConfObj.printattr()
444 parmConfObj.printattr()
445
445
446 def createObject(self, plotter_queue=None):
446 def createObject(self, plotter_queue=None):
447
447
448 if self.type == 'self':
448 if self.type == 'self':
449 raise ValueError, 'This operation type cannot be created'
449 raise ValueError, 'This operation type cannot be created'
450
450
451 if self.type == 'plotter':
451 if self.type == 'plotter':
452 if not plotter_queue:
452 if not plotter_queue:
453 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
453 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
454
454
455 opObj = Plotter(self.name, plotter_queue)
455 opObj = Plotter(self.name, plotter_queue)
456
456
457 if self.type == 'external' or self.type == 'other':
457 if self.type == 'external' or self.type == 'other':
458
458
459 className = eval(self.name)
459 className = eval(self.name)
460 kwargs = self.getKwargs()
460 kwargs = self.getKwargs()
461
461
462 opObj = className(**kwargs)
462 opObj = className(**kwargs)
463
463
464 return opObj
464 return opObj
465
465
466
466
467 class ProcUnitConf():
467 class ProcUnitConf():
468
468
469 id = None
469 id = None
470 name = None
470 name = None
471 datatype = None
471 datatype = None
472 inputId = None
472 inputId = None
473 parentId = None
473 parentId = None
474
474
475 opConfObjList = []
475 opConfObjList = []
476
476
477 procUnitObj = None
477 procUnitObj = None
478 opObjList = []
478 opObjList = []
479
479
480 ELEMENTNAME = 'ProcUnit'
480 ELEMENTNAME = 'ProcUnit'
481
481
482 def __init__(self):
482 def __init__(self):
483
483
484 self.id = None
484 self.id = None
485 self.datatype = None
485 self.datatype = None
486 self.name = None
486 self.name = None
487 self.inputId = None
487 self.inputId = None
488
488
489 self.opConfObjList = []
489 self.opConfObjList = []
490
490
491 self.procUnitObj = None
491 self.procUnitObj = None
492 self.opObjDict = {}
492 self.opObjDict = {}
493
493
494 def __getPriority(self):
494 def __getPriority(self):
495
495
496 return len(self.opConfObjList) + 1
496 return len(self.opConfObjList) + 1
497
497
498 def __getNewId(self):
498 def __getNewId(self):
499
499
500 return int(self.id) * 10 + len(self.opConfObjList) + 1
500 return int(self.id) * 10 + len(self.opConfObjList) + 1
501
501
502 def getElementName(self):
502 def getElementName(self):
503
503
504 return self.ELEMENTNAME
504 return self.ELEMENTNAME
505
505
506 def getId(self):
506 def getId(self):
507
507
508 return self.id
508 return self.id
509
509
510 def updateId(self, new_id, parentId=parentId):
510 def updateId(self, new_id, parentId=parentId):
511
511
512 new_id = int(parentId) * 10 + (int(self.id) % 10)
512 new_id = int(parentId) * 10 + (int(self.id) % 10)
513 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
513 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
514
514
515 # If this proc unit has not inputs
515 # If this proc unit has not inputs
516 if self.inputId == '0':
516 if self.inputId == '0':
517 new_inputId = 0
517 new_inputId = 0
518
518
519 n = 1
519 n = 1
520 for opConfObj in self.opConfObjList:
520 for opConfObj in self.opConfObjList:
521
521
522 idOp = str(int(new_id) * 10 + n)
522 idOp = str(int(new_id) * 10 + n)
523 opConfObj.updateId(idOp)
523 opConfObj.updateId(idOp)
524
524
525 n += 1
525 n += 1
526
526
527 self.parentId = str(parentId)
527 self.parentId = str(parentId)
528 self.id = str(new_id)
528 self.id = str(new_id)
529 self.inputId = str(new_inputId)
529 self.inputId = str(new_inputId)
530
530
531 def getInputId(self):
531 def getInputId(self):
532
532
533 return self.inputId
533 return self.inputId
534
534
535 def getOperationObjList(self):
535 def getOperationObjList(self):
536
536
537 return self.opConfObjList
537 return self.opConfObjList
538
538
539 def getOperationObj(self, name=None):
539 def getOperationObj(self, name=None):
540
540
541 for opConfObj in self.opConfObjList:
541 for opConfObj in self.opConfObjList:
542
542
543 if opConfObj.name != name:
543 if opConfObj.name != name:
544 continue
544 continue
545
545
546 return opConfObj
546 return opConfObj
547
547
548 return None
548 return None
549
549
550 def getOpObjfromParamValue(self, value=None):
550 def getOpObjfromParamValue(self, value=None):
551
551
552 for opConfObj in self.opConfObjList:
552 for opConfObj in self.opConfObjList:
553 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
553 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
554 continue
554 continue
555 return opConfObj
555 return opConfObj
556 return None
556 return None
557
557
558 def getProcUnitObj(self):
558 def getProcUnitObj(self):
559
559
560 return self.procUnitObj
560 return self.procUnitObj
561
561
562 def setup(self, id, name, datatype, inputId, parentId=None):
562 def setup(self, id, name, datatype, inputId, parentId=None):
563
563
564 # Compatible with old signal chain version
564 # Compatible with old signal chain version
565 if datatype == None and name == None:
565 if datatype == None and name == None:
566 raise ValueError, 'datatype or name should be defined'
566 raise ValueError, 'datatype or name should be defined'
567
567
568 if name == None:
568 if name == None:
569 if 'Proc' in datatype:
569 if 'Proc' in datatype:
570 name = datatype
570 name = datatype
571 else:
571 else:
572 name = '%sProc' % (datatype)
572 name = '%sProc' % (datatype)
573
573
574 if datatype == None:
574 if datatype == None:
575 datatype = name.replace('Proc', '')
575 datatype = name.replace('Proc', '')
576
576
577 self.id = str(id)
577 self.id = str(id)
578 self.name = name
578 self.name = name
579 self.datatype = datatype
579 self.datatype = datatype
580 self.inputId = inputId
580 self.inputId = inputId
581 self.parentId = parentId
581 self.parentId = parentId
582
582
583 self.opConfObjList = []
583 self.opConfObjList = []
584
584
585 self.addOperation(name='run', optype='self')
585 self.addOperation(name='run', optype='self')
586
586
587 def removeOperations(self):
587 def removeOperations(self):
588
588
589 for obj in self.opConfObjList:
589 for obj in self.opConfObjList:
590 del obj
590 del obj
591
591
592 self.opConfObjList = []
592 self.opConfObjList = []
593 self.addOperation(name='run')
593 self.addOperation(name='run')
594
594
595 def addParameter(self, **kwargs):
595 def addParameter(self, **kwargs):
596 '''
596 '''
597 Add parameters to 'run' operation
597 Add parameters to 'run' operation
598 '''
598 '''
599 opObj = self.opConfObjList[0]
599 opObj = self.opConfObjList[0]
600
600
601 opObj.addParameter(**kwargs)
601 opObj.addParameter(**kwargs)
602
602
603 return opObj
603 return opObj
604
604
605 def addOperation(self, name, optype='self'):
605 def addOperation(self, name, optype='self'):
606
606
607 id = self.__getNewId()
607 id = self.__getNewId()
608 priority = self.__getPriority()
608 priority = self.__getPriority()
609
609
610 opConfObj = OperationConf()
610 opConfObj = OperationConf()
611 opConfObj.setup(id, name=name, priority=priority, type=optype)
611 opConfObj.setup(id, name=name, priority=priority, type=optype)
612
612
613 self.opConfObjList.append(opConfObj)
613 self.opConfObjList.append(opConfObj)
614
614
615 return opConfObj
615 return opConfObj
616
616
617 def makeXml(self, projectElement):
617 def makeXml(self, projectElement):
618
618
619 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
619 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
620 procUnitElement.set('id', str(self.id))
620 procUnitElement.set('id', str(self.id))
621 procUnitElement.set('name', self.name)
621 procUnitElement.set('name', self.name)
622 procUnitElement.set('datatype', self.datatype)
622 procUnitElement.set('datatype', self.datatype)
623 procUnitElement.set('inputId', str(self.inputId))
623 procUnitElement.set('inputId', str(self.inputId))
624
624
625 for opConfObj in self.opConfObjList:
625 for opConfObj in self.opConfObjList:
626 opConfObj.makeXml(procUnitElement)
626 opConfObj.makeXml(procUnitElement)
627
627
628 def readXml(self, upElement):
628 def readXml(self, upElement):
629
629
630 self.id = upElement.get('id')
630 self.id = upElement.get('id')
631 self.name = upElement.get('name')
631 self.name = upElement.get('name')
632 self.datatype = upElement.get('datatype')
632 self.datatype = upElement.get('datatype')
633 self.inputId = upElement.get('inputId')
633 self.inputId = upElement.get('inputId')
634
634
635 if self.ELEMENTNAME == 'ReadUnit':
635 if self.ELEMENTNAME == 'ReadUnit':
636 self.datatype = self.datatype.replace('Reader', '')
636 self.datatype = self.datatype.replace('Reader', '')
637
637
638 if self.ELEMENTNAME == 'ProcUnit':
638 if self.ELEMENTNAME == 'ProcUnit':
639 self.datatype = self.datatype.replace('Proc', '')
639 self.datatype = self.datatype.replace('Proc', '')
640
640
641 if self.inputId == 'None':
641 if self.inputId == 'None':
642 self.inputId = '0'
642 self.inputId = '0'
643
643
644 self.opConfObjList = []
644 self.opConfObjList = []
645
645
646 opElementList = upElement.iter(OperationConf().getElementName())
646 opElementList = upElement.iter(OperationConf().getElementName())
647
647
648 for opElement in opElementList:
648 for opElement in opElementList:
649 opConfObj = OperationConf()
649 opConfObj = OperationConf()
650 opConfObj.readXml(opElement)
650 opConfObj.readXml(opElement)
651 self.opConfObjList.append(opConfObj)
651 self.opConfObjList.append(opConfObj)
652
652
653 def printattr(self):
653 def printattr(self):
654
654
655 print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
655 print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
656 self.id,
656 self.id,
657 self.name,
657 self.name,
658 self.datatype,
658 self.datatype,
659 self.inputId)
659 self.inputId)
660
660
661 for opConfObj in self.opConfObjList:
661 for opConfObj in self.opConfObjList:
662 opConfObj.printattr()
662 opConfObj.printattr()
663
663
664 def getKwargs(self):
664 def getKwargs(self):
665
665
666 opObj = self.opConfObjList[0]
666 opObj = self.opConfObjList[0]
667 kwargs = opObj.getKwargs()
667 kwargs = opObj.getKwargs()
668
668
669 return kwargs
669 return kwargs
670
670
671 def createObjects(self, plotter_queue=None):
671 def createObjects(self, plotter_queue=None):
672
672
673 className = eval(self.name)
673 className = eval(self.name)
674 kwargs = self.getKwargs()
674 kwargs = self.getKwargs()
675 procUnitObj = className(**kwargs)
675 procUnitObj = className(**kwargs)
676
676
677 for opConfObj in self.opConfObjList:
677 for opConfObj in self.opConfObjList:
678
678
679 if opConfObj.type == 'self' and self.name == 'run':
679 if opConfObj.type == 'self' and self.name == 'run':
680 continue
680 continue
681 elif opConfObj.type == 'self':
681 elif opConfObj.type == 'self':
682 procUnitObj.addOperationKwargs(
682 procUnitObj.addOperationKwargs(
683 opConfObj.id, **opConfObj.getKwargs())
683 opConfObj.id, **opConfObj.getKwargs())
684 continue
684 continue
685
685
686 opObj = opConfObj.createObject(plotter_queue)
686 opObj = opConfObj.createObject(plotter_queue)
687
687
688 self.opObjDict[opConfObj.id] = opObj
688 self.opObjDict[opConfObj.id] = opObj
689
689
690 procUnitObj.addOperation(opObj, opConfObj.id)
690 procUnitObj.addOperation(opObj, opConfObj.id)
691
691
692 self.procUnitObj = procUnitObj
692 self.procUnitObj = procUnitObj
693
693
694 return procUnitObj
694 return procUnitObj
695
695
696 def run(self):
696 def run(self):
697
697
698 is_ok = False
698 is_ok = False
699
699
700 for opConfObj in self.opConfObjList:
700 for opConfObj in self.opConfObjList:
701
701
702 kwargs = {}
702 kwargs = {}
703 for parmConfObj in opConfObj.getParameterObjList():
703 for parmConfObj in opConfObj.getParameterObjList():
704 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
704 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
705 continue
705 continue
706
706
707 kwargs[parmConfObj.name] = parmConfObj.getValue()
707 kwargs[parmConfObj.name] = parmConfObj.getValue()
708
708
709 sts = self.procUnitObj.call(opType=opConfObj.type,
709 sts = self.procUnitObj.call(opType=opConfObj.type,
710 opName=opConfObj.name,
710 opName=opConfObj.name,
711 opId=opConfObj.id)
711 opId=opConfObj.id)
712
712
713 is_ok = is_ok or sts
713 is_ok = is_ok or sts
714
714
715 return is_ok
715 return is_ok
716
716
717 def close(self):
717 def close(self):
718
718
719 for opConfObj in self.opConfObjList:
719 for opConfObj in self.opConfObjList:
720 if opConfObj.type == 'self':
720 if opConfObj.type == 'self':
721 continue
721 continue
722
722
723 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
723 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
724 opObj.close()
724 opObj.close()
725
725
726 self.procUnitObj.close()
726 self.procUnitObj.close()
727
727
728 return
728 return
729
729
730
730
731 class ReadUnitConf(ProcUnitConf):
731 class ReadUnitConf(ProcUnitConf):
732
732
733 path = None
733 path = None
734 startDate = None
734 startDate = None
735 endDate = None
735 endDate = None
736 startTime = None
736 startTime = None
737 endTime = None
737 endTime = None
738
738
739 ELEMENTNAME = 'ReadUnit'
739 ELEMENTNAME = 'ReadUnit'
740
740
741 def __init__(self):
741 def __init__(self):
742
742
743 self.id = None
743 self.id = None
744 self.datatype = None
744 self.datatype = None
745 self.name = None
745 self.name = None
746 self.inputId = None
746 self.inputId = None
747
747
748 self.parentId = None
748 self.parentId = None
749
749
750 self.opConfObjList = []
750 self.opConfObjList = []
751 self.opObjList = []
751 self.opObjList = []
752
752
753 def getElementName(self):
753 def getElementName(self):
754
754
755 return self.ELEMENTNAME
755 return self.ELEMENTNAME
756
756
757 def setup(self, id, name, datatype, path='', startDate='', endDate='',
757 def setup(self, id, name, datatype, path='', startDate='', endDate='',
758 startTime='', endTime='', parentId=None, server=None, **kwargs):
758 startTime='', endTime='', parentId=None, server=None, **kwargs):
759
759
760 # Compatible with old signal chain version
760 # Compatible with old signal chain version
761 if datatype == None and name == None:
761 if datatype == None and name == None:
762 raise ValueError, 'datatype or name should be defined'
762 raise ValueError, 'datatype or name should be defined'
763 if name == None:
763 if name == None:
764 if 'Reader' in datatype:
764 if 'Reader' in datatype:
765 name = datatype
765 name = datatype
766 datatype = name.replace('Reader','')
766 datatype = name.replace('Reader','')
767 else:
767 else:
768 name = '{}Reader'.format(datatype)
768 name = '{}Reader'.format(datatype)
769 if datatype == None:
769 if datatype == None:
770 if 'Reader' in name:
770 if 'Reader' in name:
771 datatype = name.replace('Reader','')
771 datatype = name.replace('Reader','')
772 else:
772 else:
773 datatype = name
773 datatype = name
774 name = '{}Reader'.format(name)
774 name = '{}Reader'.format(name)
775
775
776 self.id = id
776 self.id = id
777 self.name = name
777 self.name = name
778 self.datatype = datatype
778 self.datatype = datatype
779 if path != '':
779 if path != '':
780 self.path = os.path.abspath(path)
780 self.path = os.path.abspath(path)
781 self.startDate = startDate
781 self.startDate = startDate
782 self.endDate = endDate
782 self.endDate = endDate
783 self.startTime = startTime
783 self.startTime = startTime
784 self.endTime = endTime
784 self.endTime = endTime
785 self.inputId = '0'
785 self.inputId = '0'
786 self.parentId = parentId
786 self.parentId = parentId
787 self.server = server
787 self.server = server
788 self.addRunOperation(**kwargs)
788 self.addRunOperation(**kwargs)
789
789
790 def update(self, **kwargs):
790 def update(self, **kwargs):
791
791
792 if 'datatype' in kwargs:
792 if 'datatype' in kwargs:
793 datatype = kwargs.pop('datatype')
793 datatype = kwargs.pop('datatype')
794 if 'Reader' in datatype:
794 if 'Reader' in datatype:
795 self.name = datatype
795 self.name = datatype
796 else:
796 else:
797 self.name = '%sReader' % (datatype)
797 self.name = '%sReader' % (datatype)
798 self.datatype = self.name.replace('Reader', '')
798 self.datatype = self.name.replace('Reader', '')
799
799
800 attrs = ('path', 'startDate', 'endDate',
800 attrs = ('path', 'startDate', 'endDate',
801 'startTime', 'endTime', 'parentId')
801 'startTime', 'endTime', 'parentId')
802
802
803 for attr in attrs:
803 for attr in attrs:
804 if attr in kwargs:
804 if attr in kwargs:
805 setattr(self, attr, kwargs.pop(attr))
805 setattr(self, attr, kwargs.pop(attr))
806
806
807 self.inputId = '0'
807 self.inputId = '0'
808 self.updateRunOperation(**kwargs)
808 self.updateRunOperation(**kwargs)
809
809
810 def removeOperations(self):
810 def removeOperations(self):
811
811
812 for obj in self.opConfObjList:
812 for obj in self.opConfObjList:
813 del obj
813 del obj
814
814
815 self.opConfObjList = []
815 self.opConfObjList = []
816
816
817 def addRunOperation(self, **kwargs):
817 def addRunOperation(self, **kwargs):
818
818
819 opObj = self.addOperation(name='run', optype='self')
819 opObj = self.addOperation(name='run', optype='self')
820
820
821 if self.server is None:
821 if self.server is None:
822 opObj.addParameter(
822 opObj.addParameter(
823 name='datatype', value=self.datatype, format='str')
823 name='datatype', value=self.datatype, format='str')
824 opObj.addParameter(name='path', value=self.path, format='str')
824 opObj.addParameter(name='path', value=self.path, format='str')
825 opObj.addParameter(
825 opObj.addParameter(
826 name='startDate', value=self.startDate, format='date')
826 name='startDate', value=self.startDate, format='date')
827 opObj.addParameter(
827 opObj.addParameter(
828 name='endDate', value=self.endDate, format='date')
828 name='endDate', value=self.endDate, format='date')
829 opObj.addParameter(
829 opObj.addParameter(
830 name='startTime', value=self.startTime, format='time')
830 name='startTime', value=self.startTime, format='time')
831 opObj.addParameter(
831 opObj.addParameter(
832 name='endTime', value=self.endTime, format='time')
832 name='endTime', value=self.endTime, format='time')
833
833
834 for key, value in kwargs.items():
834 for key, value in kwargs.items():
835 opObj.addParameter(name=key, value=value,
835 opObj.addParameter(name=key, value=value,
836 format=type(value).__name__)
836 format=type(value).__name__)
837 else:
837 else:
838 opObj.addParameter(name='server', value=self.server, format='str')
838 opObj.addParameter(name='server', value=self.server, format='str')
839
839
840 return opObj
840 return opObj
841
841
842 def updateRunOperation(self, **kwargs):
842 def updateRunOperation(self, **kwargs):
843
843
844 opObj = self.getOperationObj(name='run')
844 opObj = self.getOperationObj(name='run')
845 opObj.removeParameters()
845 opObj.removeParameters()
846
846
847 opObj.addParameter(name='datatype', value=self.datatype, format='str')
847 opObj.addParameter(name='datatype', value=self.datatype, format='str')
848 opObj.addParameter(name='path', value=self.path, format='str')
848 opObj.addParameter(name='path', value=self.path, format='str')
849 opObj.addParameter(
849 opObj.addParameter(
850 name='startDate', value=self.startDate, format='date')
850 name='startDate', value=self.startDate, format='date')
851 opObj.addParameter(name='endDate', value=self.endDate, format='date')
851 opObj.addParameter(name='endDate', value=self.endDate, format='date')
852 opObj.addParameter(
852 opObj.addParameter(
853 name='startTime', value=self.startTime, format='time')
853 name='startTime', value=self.startTime, format='time')
854 opObj.addParameter(name='endTime', value=self.endTime, format='time')
854 opObj.addParameter(name='endTime', value=self.endTime, format='time')
855
855
856 for key, value in kwargs.items():
856 for key, value in kwargs.items():
857 opObj.addParameter(name=key, value=value,
857 opObj.addParameter(name=key, value=value,
858 format=type(value).__name__)
858 format=type(value).__name__)
859
859
860 return opObj
860 return opObj
861
861
862 def readXml(self, upElement):
862 def readXml(self, upElement):
863
863
864 self.id = upElement.get('id')
864 self.id = upElement.get('id')
865 self.name = upElement.get('name')
865 self.name = upElement.get('name')
866 self.datatype = upElement.get('datatype')
866 self.datatype = upElement.get('datatype')
867 self.inputId = upElement.get('inputId')
867 self.inputId = upElement.get('inputId')
868
868
869 if self.ELEMENTNAME == 'ReadUnit':
869 if self.ELEMENTNAME == 'ReadUnit':
870 self.datatype = self.datatype.replace('Reader', '')
870 self.datatype = self.datatype.replace('Reader', '')
871
871
872 if self.inputId == 'None':
872 if self.inputId == 'None':
873 self.inputId = '0'
873 self.inputId = '0'
874
874
875 self.opConfObjList = []
875 self.opConfObjList = []
876
876
877 opElementList = upElement.iter(OperationConf().getElementName())
877 opElementList = upElement.iter(OperationConf().getElementName())
878
878
879 for opElement in opElementList:
879 for opElement in opElementList:
880 opConfObj = OperationConf()
880 opConfObj = OperationConf()
881 opConfObj.readXml(opElement)
881 opConfObj.readXml(opElement)
882 self.opConfObjList.append(opConfObj)
882 self.opConfObjList.append(opConfObj)
883
883
884 if opConfObj.name == 'run':
884 if opConfObj.name == 'run':
885 self.path = opConfObj.getParameterValue('path')
885 self.path = opConfObj.getParameterValue('path')
886 self.startDate = opConfObj.getParameterValue('startDate')
886 self.startDate = opConfObj.getParameterValue('startDate')
887 self.endDate = opConfObj.getParameterValue('endDate')
887 self.endDate = opConfObj.getParameterValue('endDate')
888 self.startTime = opConfObj.getParameterValue('startTime')
888 self.startTime = opConfObj.getParameterValue('startTime')
889 self.endTime = opConfObj.getParameterValue('endTime')
889 self.endTime = opConfObj.getParameterValue('endTime')
890
890
891
891
892 class Project(Process):
892 class Project(Process):
893
893
894 id = None
894 id = None
895 # name = None
895 # name = None
896 description = None
896 description = None
897 filename = None
897 filename = None
898
898
899 procUnitConfObjDict = None
899 procUnitConfObjDict = None
900
900
901 ELEMENTNAME = 'Project'
901 ELEMENTNAME = 'Project'
902
902
903 plotterQueue = None
903 plotterQueue = None
904
904
905 def __init__(self, plotter_queue=None):
905 def __init__(self, plotter_queue=None):
906
906
907 Process.__init__(self)
907 Process.__init__(self)
908 self.id = None
908 self.id = None
909 self.description = None
909 self.description = None
910 self.email = None
910 self.email = None
911 self.alarm = [0]
911 self.alarm = [0]
912 self.plotterQueue = plotter_queue
912 self.plotterQueue = plotter_queue
913 self.procUnitConfObjDict = {}
913 self.procUnitConfObjDict = {}
914
914
915 def __getNewId(self):
915 def __getNewId(self):
916
916
917 idList = self.procUnitConfObjDict.keys()
917 idList = self.procUnitConfObjDict.keys()
918
918
919 id = int(self.id) * 10
919 id = int(self.id) * 10
920
920
921 while True:
921 while True:
922 id += 1
922 id += 1
923
923
924 if str(id) in idList:
924 if str(id) in idList:
925 continue
925 continue
926
926
927 break
927 break
928
928
929 return str(id)
929 return str(id)
930
930
931 def getElementName(self):
931 def getElementName(self):
932
932
933 return self.ELEMENTNAME
933 return self.ELEMENTNAME
934
934
935 def getId(self):
935 def getId(self):
936
936
937 return self.id
937 return self.id
938
938
939 def updateId(self, new_id):
939 def updateId(self, new_id):
940
940
941 self.id = str(new_id)
941 self.id = str(new_id)
942
942
943 keyList = self.procUnitConfObjDict.keys()
943 keyList = self.procUnitConfObjDict.keys()
944 keyList.sort()
944 keyList.sort()
945
945
946 n = 1
946 n = 1
947 newProcUnitConfObjDict = {}
947 newProcUnitConfObjDict = {}
948
948
949 for procKey in keyList:
949 for procKey in keyList:
950
950
951 procUnitConfObj = self.procUnitConfObjDict[procKey]
951 procUnitConfObj = self.procUnitConfObjDict[procKey]
952 idProcUnit = str(int(self.id) * 10 + n)
952 idProcUnit = str(int(self.id) * 10 + n)
953 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
953 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
954 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
954 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
955 n += 1
955 n += 1
956
956
957 self.procUnitConfObjDict = newProcUnitConfObjDict
957 self.procUnitConfObjDict = newProcUnitConfObjDict
958
958
959 def setup(self, id, name='', description='', email=None, alarm=[0]):
959 def setup(self, id, name='', description='', email=None, alarm=[0]):
960
960
961 print
961 print
962 print '*' * 60
962 print '*' * 60
963 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
963 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
964 print '*' * 60
964 print '*' * 60
965 print
965 print
966 self.id = str(id)
966 self.id = str(id)
967 self.description = description
967 self.description = description
968 self.email = email
968 self.email = email
969 self.alarm = alarm
969 self.alarm = alarm
970
970
971 def update(self, **kwargs):
971 def update(self, **kwargs):
972
972
973 for key, value in kwargs:
973 for key, value in kwargs.items():
974 setattr(self, key, value)
974 setattr(self, key, value)
975
975
976 def clone(self):
976 def clone(self):
977
977
978 p = Project()
978 p = Project()
979 p.procUnitConfObjDict = self.procUnitConfObjDict
979 p.procUnitConfObjDict = self.procUnitConfObjDict
980 return p
980 return p
981
981
982 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
982 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
983
983
984 if id is None:
984 if id is None:
985 idReadUnit = self.__getNewId()
985 idReadUnit = self.__getNewId()
986 else:
986 else:
987 idReadUnit = str(id)
987 idReadUnit = str(id)
988
988
989 readUnitConfObj = ReadUnitConf()
989 readUnitConfObj = ReadUnitConf()
990 readUnitConfObj.setup(idReadUnit, name, datatype,
990 readUnitConfObj.setup(idReadUnit, name, datatype,
991 parentId=self.id, **kwargs)
991 parentId=self.id, **kwargs)
992
992
993 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
993 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
994
994
995 return readUnitConfObj
995 return readUnitConfObj
996
996
997 def addProcUnit(self, inputId='0', datatype=None, name=None):
997 def addProcUnit(self, inputId='0', datatype=None, name=None):
998
998
999 idProcUnit = self.__getNewId()
999 idProcUnit = self.__getNewId()
1000
1000
1001 procUnitConfObj = ProcUnitConf()
1001 procUnitConfObj = ProcUnitConf()
1002 procUnitConfObj.setup(idProcUnit, name, datatype,
1002 procUnitConfObj.setup(idProcUnit, name, datatype,
1003 inputId, parentId=self.id)
1003 inputId, parentId=self.id)
1004
1004
1005 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1005 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1006
1006
1007 return procUnitConfObj
1007 return procUnitConfObj
1008
1008
1009 def removeProcUnit(self, id):
1009 def removeProcUnit(self, id):
1010
1010
1011 if id in self.procUnitConfObjDict.keys():
1011 if id in self.procUnitConfObjDict.keys():
1012 self.procUnitConfObjDict.pop(id)
1012 self.procUnitConfObjDict.pop(id)
1013
1013
1014 def getReadUnitId(self):
1014 def getReadUnitId(self):
1015
1015
1016 readUnitConfObj = self.getReadUnitObj()
1016 readUnitConfObj = self.getReadUnitObj()
1017
1017
1018 return readUnitConfObj.id
1018 return readUnitConfObj.id
1019
1019
1020 def getReadUnitObj(self):
1020 def getReadUnitObj(self):
1021
1021
1022 for obj in self.procUnitConfObjDict.values():
1022 for obj in self.procUnitConfObjDict.values():
1023 if obj.getElementName() == 'ReadUnit':
1023 if obj.getElementName() == 'ReadUnit':
1024 return obj
1024 return obj
1025
1025
1026 return None
1026 return None
1027
1027
1028 def getProcUnitObj(self, id=None, name=None):
1028 def getProcUnitObj(self, id=None, name=None):
1029
1029
1030 if id != None:
1030 if id != None:
1031 return self.procUnitConfObjDict[id]
1031 return self.procUnitConfObjDict[id]
1032
1032
1033 if name != None:
1033 if name != None:
1034 return self.getProcUnitObjByName(name)
1034 return self.getProcUnitObjByName(name)
1035
1035
1036 return None
1036 return None
1037
1037
1038 def getProcUnitObjByName(self, name):
1038 def getProcUnitObjByName(self, name):
1039
1039
1040 for obj in self.procUnitConfObjDict.values():
1040 for obj in self.procUnitConfObjDict.values():
1041 if obj.name == name:
1041 if obj.name == name:
1042 return obj
1042 return obj
1043
1043
1044 return None
1044 return None
1045
1045
1046 def procUnitItems(self):
1046 def procUnitItems(self):
1047
1047
1048 return self.procUnitConfObjDict.items()
1048 return self.procUnitConfObjDict.items()
1049
1049
1050 def makeXml(self):
1050 def makeXml(self):
1051
1051
1052 projectElement = Element('Project')
1052 projectElement = Element('Project')
1053 projectElement.set('id', str(self.id))
1053 projectElement.set('id', str(self.id))
1054 projectElement.set('name', self.name)
1054 projectElement.set('name', self.name)
1055 projectElement.set('description', self.description)
1055 projectElement.set('description', self.description)
1056
1056
1057 for procUnitConfObj in self.procUnitConfObjDict.values():
1057 for procUnitConfObj in self.procUnitConfObjDict.values():
1058 procUnitConfObj.makeXml(projectElement)
1058 procUnitConfObj.makeXml(projectElement)
1059
1059
1060 self.projectElement = projectElement
1060 self.projectElement = projectElement
1061
1061
1062 def writeXml(self, filename=None):
1062 def writeXml(self, filename=None):
1063
1063
1064 if filename == None:
1064 if filename == None:
1065 if self.filename:
1065 if self.filename:
1066 filename = self.filename
1066 filename = self.filename
1067 else:
1067 else:
1068 filename = 'schain.xml'
1068 filename = 'schain.xml'
1069
1069
1070 if not filename:
1070 if not filename:
1071 print 'filename has not been defined. Use setFilename(filename) for do it.'
1071 print 'filename has not been defined. Use setFilename(filename) for do it.'
1072 return 0
1072 return 0
1073
1073
1074 abs_file = os.path.abspath(filename)
1074 abs_file = os.path.abspath(filename)
1075
1075
1076 if not os.access(os.path.dirname(abs_file), os.W_OK):
1076 if not os.access(os.path.dirname(abs_file), os.W_OK):
1077 print 'No write permission on %s' % os.path.dirname(abs_file)
1077 print 'No write permission on %s' % os.path.dirname(abs_file)
1078 return 0
1078 return 0
1079
1079
1080 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1080 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1081 print 'File %s already exists and it could not be overwriten' % abs_file
1081 print 'File %s already exists and it could not be overwriten' % abs_file
1082 return 0
1082 return 0
1083
1083
1084 self.makeXml()
1084 self.makeXml()
1085
1085
1086 ElementTree(self.projectElement).write(abs_file, method='xml')
1086 ElementTree(self.projectElement).write(abs_file, method='xml')
1087
1087
1088 self.filename = abs_file
1088 self.filename = abs_file
1089
1089
1090 return 1
1090 return 1
1091
1091
1092 def readXml(self, filename=None):
1092 def readXml(self, filename=None):
1093
1093
1094 if not filename:
1094 if not filename:
1095 print 'filename is not defined'
1095 print 'filename is not defined'
1096 return 0
1096 return 0
1097
1097
1098 abs_file = os.path.abspath(filename)
1098 abs_file = os.path.abspath(filename)
1099
1099
1100 if not os.path.isfile(abs_file):
1100 if not os.path.isfile(abs_file):
1101 print '%s file does not exist' % abs_file
1101 print '%s file does not exist' % abs_file
1102 return 0
1102 return 0
1103
1103
1104 self.projectElement = None
1104 self.projectElement = None
1105 self.procUnitConfObjDict = {}
1105 self.procUnitConfObjDict = {}
1106
1106
1107 try:
1107 try:
1108 self.projectElement = ElementTree().parse(abs_file)
1108 self.projectElement = ElementTree().parse(abs_file)
1109 except:
1109 except:
1110 print 'Error reading %s, verify file format' % filename
1110 print 'Error reading %s, verify file format' % filename
1111 return 0
1111 return 0
1112
1112
1113 self.project = self.projectElement.tag
1113 self.project = self.projectElement.tag
1114
1114
1115 self.id = self.projectElement.get('id')
1115 self.id = self.projectElement.get('id')
1116 self.name = self.projectElement.get('name')
1116 self.name = self.projectElement.get('name')
1117 self.description = self.projectElement.get('description')
1117 self.description = self.projectElement.get('description')
1118
1118
1119 readUnitElementList = self.projectElement.iter(
1119 readUnitElementList = self.projectElement.iter(
1120 ReadUnitConf().getElementName())
1120 ReadUnitConf().getElementName())
1121
1121
1122 for readUnitElement in readUnitElementList:
1122 for readUnitElement in readUnitElementList:
1123 readUnitConfObj = ReadUnitConf()
1123 readUnitConfObj = ReadUnitConf()
1124 readUnitConfObj.readXml(readUnitElement)
1124 readUnitConfObj.readXml(readUnitElement)
1125
1125
1126 if readUnitConfObj.parentId == None:
1126 if readUnitConfObj.parentId == None:
1127 readUnitConfObj.parentId = self.id
1127 readUnitConfObj.parentId = self.id
1128
1128
1129 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1129 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1130
1130
1131 procUnitElementList = self.projectElement.iter(
1131 procUnitElementList = self.projectElement.iter(
1132 ProcUnitConf().getElementName())
1132 ProcUnitConf().getElementName())
1133
1133
1134 for procUnitElement in procUnitElementList:
1134 for procUnitElement in procUnitElementList:
1135 procUnitConfObj = ProcUnitConf()
1135 procUnitConfObj = ProcUnitConf()
1136 procUnitConfObj.readXml(procUnitElement)
1136 procUnitConfObj.readXml(procUnitElement)
1137
1137
1138 if procUnitConfObj.parentId == None:
1138 if procUnitConfObj.parentId == None:
1139 procUnitConfObj.parentId = self.id
1139 procUnitConfObj.parentId = self.id
1140
1140
1141 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1141 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1142
1142
1143 self.filename = abs_file
1143 self.filename = abs_file
1144
1144
1145 return 1
1145 return 1
1146
1146
1147 def printattr(self):
1147 def printattr(self):
1148
1148
1149 print 'Project[%s]: name = %s, description = %s' % (self.id,
1149 print 'Project[%s]: name = %s, description = %s' % (self.id,
1150 self.name,
1150 self.name,
1151 self.description)
1151 self.description)
1152
1152
1153 for procUnitConfObj in self.procUnitConfObjDict.values():
1153 for procUnitConfObj in self.procUnitConfObjDict.values():
1154 procUnitConfObj.printattr()
1154 procUnitConfObj.printattr()
1155
1155
1156 def createObjects(self):
1156 def createObjects(self):
1157
1157
1158 for procUnitConfObj in self.procUnitConfObjDict.values():
1158 for procUnitConfObj in self.procUnitConfObjDict.values():
1159 procUnitConfObj.createObjects(self.plotterQueue)
1159 procUnitConfObj.createObjects(self.plotterQueue)
1160
1160
1161 def __connect(self, objIN, thisObj):
1161 def __connect(self, objIN, thisObj):
1162
1162
1163 thisObj.setInput(objIN.getOutputObj())
1163 thisObj.setInput(objIN.getOutputObj())
1164
1164
1165 def connectObjects(self):
1165 def connectObjects(self):
1166
1166
1167 for thisPUConfObj in self.procUnitConfObjDict.values():
1167 for thisPUConfObj in self.procUnitConfObjDict.values():
1168
1168
1169 inputId = thisPUConfObj.getInputId()
1169 inputId = thisPUConfObj.getInputId()
1170
1170
1171 if int(inputId) == 0:
1171 if int(inputId) == 0:
1172 continue
1172 continue
1173
1173
1174 # Get input object
1174 # Get input object
1175 puConfINObj = self.procUnitConfObjDict[inputId]
1175 puConfINObj = self.procUnitConfObjDict[inputId]
1176 puObjIN = puConfINObj.getProcUnitObj()
1176 puObjIN = puConfINObj.getProcUnitObj()
1177
1177
1178 # Get current object
1178 # Get current object
1179 thisPUObj = thisPUConfObj.getProcUnitObj()
1179 thisPUObj = thisPUConfObj.getProcUnitObj()
1180
1180
1181 self.__connect(puObjIN, thisPUObj)
1181 self.__connect(puObjIN, thisPUObj)
1182
1182
1183 def __handleError(self, procUnitConfObj, modes=None):
1183 def __handleError(self, procUnitConfObj, modes=None):
1184
1184
1185 import socket
1185 import socket
1186
1186
1187 if modes is None:
1187 if modes is None:
1188 modes = self.alarm
1188 modes = self.alarm
1189
1189
1190 err = traceback.format_exception(sys.exc_info()[0],
1190 err = traceback.format_exception(sys.exc_info()[0],
1191 sys.exc_info()[1],
1191 sys.exc_info()[1],
1192 sys.exc_info()[2])
1192 sys.exc_info()[2])
1193
1193
1194 log.error('{}'.format(err[-1]), procUnitConfObj.name)
1194 log.error('{}'.format(err[-1]), procUnitConfObj.name)
1195
1195
1196 message = ''.join(err)
1196 message = ''.join(err)
1197
1197
1198 sys.stderr.write(message)
1198 sys.stderr.write(message)
1199
1199
1200 subject = 'SChain v%s: Error running %s\n' % (
1200 subject = 'SChain v%s: Error running %s\n' % (
1201 schainpy.__version__, procUnitConfObj.name)
1201 schainpy.__version__, procUnitConfObj.name)
1202
1202
1203 subtitle = '%s: %s\n' % (
1203 subtitle = '%s: %s\n' % (
1204 procUnitConfObj.getElementName(), procUnitConfObj.name)
1204 procUnitConfObj.getElementName(), procUnitConfObj.name)
1205 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1205 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1206 socket.gethostname())
1206 socket.gethostname())
1207 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1207 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1208 subtitle += 'Configuration file: %s\n' % self.filename
1208 subtitle += 'Configuration file: %s\n' % self.filename
1209 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1209 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1210
1210
1211 readUnitConfObj = self.getReadUnitObj()
1211 readUnitConfObj = self.getReadUnitObj()
1212 if readUnitConfObj:
1212 if readUnitConfObj:
1213 subtitle += '\nInput parameters:\n'
1213 subtitle += '\nInput parameters:\n'
1214 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1214 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1215 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1215 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1216 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1216 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1217 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1217 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1218 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1218 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1219 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1219 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1220
1220
1221 a = Alarm(
1221 a = Alarm(
1222 modes=modes,
1222 modes=modes,
1223 email=self.email,
1223 email=self.email,
1224 message=message,
1224 message=message,
1225 subject=subject,
1225 subject=subject,
1226 subtitle=subtitle,
1226 subtitle=subtitle,
1227 filename=self.filename
1227 filename=self.filename
1228 )
1228 )
1229
1229
1230 a.start()
1230 a.start()
1231
1231
1232 def isPaused(self):
1232 def isPaused(self):
1233 return 0
1233 return 0
1234
1234
1235 def isStopped(self):
1235 def isStopped(self):
1236 return 0
1236 return 0
1237
1237
1238 def runController(self):
1238 def runController(self):
1239 '''
1239 '''
1240 returns 0 when this process has been stopped, 1 otherwise
1240 returns 0 when this process has been stopped, 1 otherwise
1241 '''
1241 '''
1242
1242
1243 if self.isPaused():
1243 if self.isPaused():
1244 print 'Process suspended'
1244 print 'Process suspended'
1245
1245
1246 while True:
1246 while True:
1247 time.sleep(0.1)
1247 time.sleep(0.1)
1248
1248
1249 if not self.isPaused():
1249 if not self.isPaused():
1250 break
1250 break
1251
1251
1252 if self.isStopped():
1252 if self.isStopped():
1253 break
1253 break
1254
1254
1255 print 'Process reinitialized'
1255 print 'Process reinitialized'
1256
1256
1257 if self.isStopped():
1257 if self.isStopped():
1258 print 'Process stopped'
1258 print 'Process stopped'
1259 return 0
1259 return 0
1260
1260
1261 return 1
1261 return 1
1262
1262
1263 def setFilename(self, filename):
1263 def setFilename(self, filename):
1264
1264
1265 self.filename = filename
1265 self.filename = filename
1266
1266
1267 def setPlotterQueue(self, plotter_queue):
1267 def setPlotterQueue(self, plotter_queue):
1268
1268
1269 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1269 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1270
1270
1271 def getPlotterQueue(self):
1271 def getPlotterQueue(self):
1272
1272
1273 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1273 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1274
1274
1275 def useExternalPlotter(self):
1275 def useExternalPlotter(self):
1276
1276
1277 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1277 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1278
1278
1279 def run(self):
1279 def run(self):
1280
1280
1281 log.success('Starting {}'.format(self.name))
1281 log.success('Starting {}'.format(self.name))
1282 self.start_time = time.time()
1282 self.start_time = time.time()
1283 self.createObjects()
1283 self.createObjects()
1284 self.connectObjects()
1284 self.connectObjects()
1285
1285
1286 keyList = self.procUnitConfObjDict.keys()
1286 keyList = self.procUnitConfObjDict.keys()
1287 keyList.sort()
1287 keyList.sort()
1288
1288
1289 while(True):
1289 while(True):
1290
1290
1291 is_ok = False
1291 is_ok = False
1292
1292
1293 for procKey in keyList:
1293 for procKey in keyList:
1294
1294
1295 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295 procUnitConfObj = self.procUnitConfObjDict[procKey]
1296
1296
1297 try:
1297 try:
1298 sts = procUnitConfObj.run()
1298 sts = procUnitConfObj.run()
1299 is_ok = is_ok or sts
1299 is_ok = is_ok or sts
1300 except SchainWarning:
1300 except SchainWarning:
1301 self.__handleError(procUnitConfObj, modes=[2, 3])
1301 self.__handleError(procUnitConfObj, modes=[2, 3])
1302 except KeyboardInterrupt:
1302 except KeyboardInterrupt:
1303 is_ok = False
1303 is_ok = False
1304 break
1304 break
1305 except ValueError, e:
1305 except ValueError, e:
1306 time.sleep(0.5)
1306 time.sleep(0.5)
1307 self.__handleError(procUnitConfObj)
1307 self.__handleError(procUnitConfObj)
1308 is_ok = False
1308 is_ok = False
1309 break
1309 break
1310 except:
1310 except:
1311 time.sleep(0.5)
1311 time.sleep(0.5)
1312 self.__handleError(procUnitConfObj)
1312 self.__handleError(procUnitConfObj)
1313 is_ok = False
1313 is_ok = False
1314 break
1314 break
1315
1315
1316 # If every process unit finished so end process
1316 # If every process unit finished so end process
1317 if not(is_ok):
1317 if not(is_ok):
1318 break
1318 break
1319
1319
1320 if not self.runController():
1320 if not self.runController():
1321 break
1321 break
1322
1322
1323 # Closing every process
1323 # Closing every process
1324 for procKey in keyList:
1324 for procKey in keyList:
1325 procUnitConfObj = self.procUnitConfObjDict[procKey]
1325 procUnitConfObj = self.procUnitConfObjDict[procKey]
1326 procUnitConfObj.close()
1326 procUnitConfObj.close()
1327
1327
1328 log.success('{} finished (time: {}s)'.format(
1328 log.success('{} finished (time: {}s)'.format(
1329 self.name,
1329 self.name,
1330 time.time()-self.start_time))
1330 time.time()-self.start_time))
General Comments 0
You need to be logged in to leave comments. Login now