##// END OF EJS Templates
schain
RSS
quitado el logger
José Chávez -
r1049:5338a74c35d8
parent child
Show More
Side by Side Unified
Context file:
r1049:5338a74c35d8 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1330 +1,1324
1 '''
1 '''
2 Created on September , 2012
2 Created on September , 2012
3 @author:
3 @author:
4 '''
4 '''
5
5
6 import sys
6 import sys
7
8 # save_stdout = sys.stdout
9 # logToFile = newLogger()
10 # sys.stdout = logToFile
11 # sys.stderr = logToFile
12
13 import ast
7 import ast
14 import datetime
8 import datetime
15 import traceback
9 import traceback
16 import math
10 import math
17 import time
11 import time
18 from multiprocessing import Process, Queue, cpu_count
12 from multiprocessing import Process, Queue, cpu_count
19
13
20 import schainpy
14 import schainpy
21 import schainpy.admin
15 import schainpy.admin
22 from schainpy.utils.log import logToFile
16 from schainpy.utils.log import logToFile
23
17
24 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
18 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
25 from xml.dom import minidom
19 from xml.dom import minidom
26
20
27 from schainpy.model import *
21 from schainpy.model import *
28 from time import sleep
22 from time import sleep
29
23
30
24
31
25
32 def prettify(elem):
26 def prettify(elem):
33 """Return a pretty-printed XML string for the Element.
27 """Return a pretty-printed XML string for the Element.
34 """
28 """
35 rough_string = tostring(elem, 'utf-8')
29 rough_string = tostring(elem, 'utf-8')
36 reparsed = minidom.parseString(rough_string)
30 reparsed = minidom.parseString(rough_string)
37 return reparsed.toprettyxml(indent=" ")
31 return reparsed.toprettyxml(indent=" ")
38
32
39 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
33 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
40 skip = 0
34 skip = 0
41 cursor = 0
35 cursor = 0
42 nFiles = None
36 nFiles = None
43 processes = []
37 processes = []
44 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
38 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
45 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
39 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
46 days = (dt2 - dt1).days
40 days = (dt2 - dt1).days
47
41
48 for day in range(days+1):
42 for day in range(days+1):
49 skip = 0
43 skip = 0
50 cursor = 0
44 cursor = 0
51 q = Queue()
45 q = Queue()
52 processes = []
46 processes = []
53 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
47 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
54 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
48 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
55 firstProcess.start()
49 firstProcess.start()
56 if by_day:
50 if by_day:
57 continue
51 continue
58 nFiles = q.get()
52 nFiles = q.get()
59 if nFiles==0:
53 if nFiles==0:
60 continue
54 continue
61 firstProcess.terminate()
55 firstProcess.terminate()
62 skip = int(math.ceil(nFiles/nProcess))
56 skip = int(math.ceil(nFiles/nProcess))
63 while True:
57 while True:
64 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
58 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
65 processes[cursor].start()
59 processes[cursor].start()
66 if nFiles < cursor*skip:
60 if nFiles < cursor*skip:
67 break
61 break
68 cursor += 1
62 cursor += 1
69
63
70 def beforeExit(exctype, value, trace):
64 def beforeExit(exctype, value, trace):
71 for process in processes:
65 for process in processes:
72 process.terminate()
66 process.terminate()
73 process.join()
67 process.join()
74 print traceback.print_tb(trace)
68 print traceback.print_tb(trace)
75
69
76 sys.excepthook = beforeExit
70 sys.excepthook = beforeExit
77
71
78 for process in processes:
72 for process in processes:
79 process.join()
73 process.join()
80 process.terminate()
74 process.terminate()
81
75
82 time.sleep(3)
76 time.sleep(3)
83
77
84
78
85 class ParameterConf():
79 class ParameterConf():
86
80
87 id = None
81 id = None
88 name = None
82 name = None
89 value = None
83 value = None
90 format = None
84 format = None
91
85
92 __formated_value = None
86 __formated_value = None
93
87
94 ELEMENTNAME = 'Parameter'
88 ELEMENTNAME = 'Parameter'
95
89
96 def __init__(self):
90 def __init__(self):
97
91
98 self.format = 'str'
92 self.format = 'str'
99
93
100 def getElementName(self):
94 def getElementName(self):
101
95
102 return self.ELEMENTNAME
96 return self.ELEMENTNAME
103
97
104 def getValue(self):
98 def getValue(self):
105
99
106 value = self.value
100 value = self.value
107 format = self.format
101 format = self.format
108
102
109 if self.__formated_value != None:
103 if self.__formated_value != None:
110
104
111 return self.__formated_value
105 return self.__formated_value
112
106
113 if format == 'obj':
107 if format == 'obj':
114 return value
108 return value
115
109
116 if format == 'str':
110 if format == 'str':
117 self.__formated_value = str(value)
111 self.__formated_value = str(value)
118 return self.__formated_value
112 return self.__formated_value
119
113
120 if value == '':
114 if value == '':
121 raise ValueError, "%s: This parameter value is empty" %self.name
115 raise ValueError, "%s: This parameter value is empty" %self.name
122
116
123 if format == 'list':
117 if format == 'list':
124 strList = value.split(',')
118 strList = value.split(',')
125
119
126 self.__formated_value = strList
120 self.__formated_value = strList
127
121
128 return self.__formated_value
122 return self.__formated_value
129
123
130 if format == 'intlist':
124 if format == 'intlist':
131 """
125 """
132 Example:
126 Example:
133 value = (0,1,2)
127 value = (0,1,2)
134 """
128 """
135
129
136 new_value = ast.literal_eval(value)
130 new_value = ast.literal_eval(value)
137
131
138 if type(new_value) not in (tuple, list):
132 if type(new_value) not in (tuple, list):
139 new_value = [int(new_value)]
133 new_value = [int(new_value)]
140
134
141 self.__formated_value = new_value
135 self.__formated_value = new_value
142
136
143 return self.__formated_value
137 return self.__formated_value
144
138
145 if format == 'floatlist':
139 if format == 'floatlist':
146 """
140 """
147 Example:
141 Example:
148 value = (0.5, 1.4, 2.7)
142 value = (0.5, 1.4, 2.7)
149 """
143 """
150
144
151 new_value = ast.literal_eval(value)
145 new_value = ast.literal_eval(value)
152
146
153 if type(new_value) not in (tuple, list):
147 if type(new_value) not in (tuple, list):
154 new_value = [float(new_value)]
148 new_value = [float(new_value)]
155
149
156 self.__formated_value = new_value
150 self.__formated_value = new_value
157
151
158 return self.__formated_value
152 return self.__formated_value
159
153
160 if format == 'date':
154 if format == 'date':
161 strList = value.split('/')
155 strList = value.split('/')
162 intList = [int(x) for x in strList]
156 intList = [int(x) for x in strList]
163 date = datetime.date(intList[0], intList[1], intList[2])
157 date = datetime.date(intList[0], intList[1], intList[2])
164
158
165 self.__formated_value = date
159 self.__formated_value = date
166
160
167 return self.__formated_value
161 return self.__formated_value
168
162
169 if format == 'time':
163 if format == 'time':
170 strList = value.split(':')
164 strList = value.split(':')
171 intList = [int(x) for x in strList]
165 intList = [int(x) for x in strList]
172 time = datetime.time(intList[0], intList[1], intList[2])
166 time = datetime.time(intList[0], intList[1], intList[2])
173
167
174 self.__formated_value = time
168 self.__formated_value = time
175
169
176 return self.__formated_value
170 return self.__formated_value
177
171
178 if format == 'pairslist':
172 if format == 'pairslist':
179 """
173 """
180 Example:
174 Example:
181 value = (0,1),(1,2)
175 value = (0,1),(1,2)
182 """
176 """
183
177
184 new_value = ast.literal_eval(value)
178 new_value = ast.literal_eval(value)
185
179
186 if type(new_value) not in (tuple, list):
180 if type(new_value) not in (tuple, list):
187 raise ValueError, "%s has to be a tuple or list of pairs" %value
181 raise ValueError, "%s has to be a tuple or list of pairs" %value
188
182
189 if type(new_value[0]) not in (tuple, list):
183 if type(new_value[0]) not in (tuple, list):
190 if len(new_value) != 2:
184 if len(new_value) != 2:
191 raise ValueError, "%s has to be a tuple or list of pairs" %value
185 raise ValueError, "%s has to be a tuple or list of pairs" %value
192 new_value = [new_value]
186 new_value = [new_value]
193
187
194 for thisPair in new_value:
188 for thisPair in new_value:
195 if len(thisPair) != 2:
189 if len(thisPair) != 2:
196 raise ValueError, "%s has to be a tuple or list of pairs" %value
190 raise ValueError, "%s has to be a tuple or list of pairs" %value
197
191
198 self.__formated_value = new_value
192 self.__formated_value = new_value
199
193
200 return self.__formated_value
194 return self.__formated_value
201
195
202 if format == 'multilist':
196 if format == 'multilist':
203 """
197 """
204 Example:
198 Example:
205 value = (0,1,2),(3,4,5)
199 value = (0,1,2),(3,4,5)
206 """
200 """
207 multiList = ast.literal_eval(value)
201 multiList = ast.literal_eval(value)
208
202
209 if type(multiList[0]) == int:
203 if type(multiList[0]) == int:
210 multiList = ast.literal_eval("(" + value + ")")
204 multiList = ast.literal_eval("(" + value + ")")
211
205
212 self.__formated_value = multiList
206 self.__formated_value = multiList
213
207
214 return self.__formated_value
208 return self.__formated_value
215
209
216 if format == 'bool':
210 if format == 'bool':
217 value = int(value)
211 value = int(value)
218
212
219 if format == 'int':
213 if format == 'int':
220 value = float(value)
214 value = float(value)
221
215
222 format_func = eval(format)
216 format_func = eval(format)
223
217
224 self.__formated_value = format_func(value)
218 self.__formated_value = format_func(value)
225
219
226 return self.__formated_value
220 return self.__formated_value
227
221
228 def updateId(self, new_id):
222 def updateId(self, new_id):
229
223
230 self.id = str(new_id)
224 self.id = str(new_id)
231
225
232 def setup(self, id, name, value, format='str'):
226 def setup(self, id, name, value, format='str'):
233 self.id = str(id)
227 self.id = str(id)
234 self.name = name
228 self.name = name
235 if format == 'obj':
229 if format == 'obj':
236 self.value = value
230 self.value = value
237 else:
231 else:
238 self.value = str(value)
232 self.value = str(value)
239 self.format = str.lower(format)
233 self.format = str.lower(format)
240
234
241 self.getValue()
235 self.getValue()
242
236
243 return 1
237 return 1
244
238
245 def update(self, name, value, format='str'):
239 def update(self, name, value, format='str'):
246
240
247 self.name = name
241 self.name = name
248 self.value = str(value)
242 self.value = str(value)
249 self.format = format
243 self.format = format
250
244
251 def makeXml(self, opElement):
245 def makeXml(self, opElement):
252 if self.name not in ('queue',):
246 if self.name not in ('queue',):
253 parmElement = SubElement(opElement, self.ELEMENTNAME)
247 parmElement = SubElement(opElement, self.ELEMENTNAME)
254 parmElement.set('id', str(self.id))
248 parmElement.set('id', str(self.id))
255 parmElement.set('name', self.name)
249 parmElement.set('name', self.name)
256 parmElement.set('value', self.value)
250 parmElement.set('value', self.value)
257 parmElement.set('format', self.format)
251 parmElement.set('format', self.format)
258
252
259 def readXml(self, parmElement):
253 def readXml(self, parmElement):
260
254
261 self.id = parmElement.get('id')
255 self.id = parmElement.get('id')
262 self.name = parmElement.get('name')
256 self.name = parmElement.get('name')
263 self.value = parmElement.get('value')
257 self.value = parmElement.get('value')
264 self.format = str.lower(parmElement.get('format'))
258 self.format = str.lower(parmElement.get('format'))
265
259
266 #Compatible with old signal chain version
260 #Compatible with old signal chain version
267 if self.format == 'int' and self.name == 'idfigure':
261 if self.format == 'int' and self.name == 'idfigure':
268 self.name = 'id'
262 self.name = 'id'
269
263
270 def printattr(self):
264 def printattr(self):
271
265
272 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
266 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
273
267
274 class OperationConf():
268 class OperationConf():
275
269
276 id = None
270 id = None
277 name = None
271 name = None
278 priority = None
272 priority = None
279 type = None
273 type = None
280
274
281 parmConfObjList = []
275 parmConfObjList = []
282
276
283 ELEMENTNAME = 'Operation'
277 ELEMENTNAME = 'Operation'
284
278
285 def __init__(self):
279 def __init__(self):
286
280
287 self.id = '0'
281 self.id = '0'
288 self.name = None
282 self.name = None
289 self.priority = None
283 self.priority = None
290 self.type = 'self'
284 self.type = 'self'
291
285
292
286
293 def __getNewId(self):
287 def __getNewId(self):
294
288
295 return int(self.id)*10 + len(self.parmConfObjList) + 1
289 return int(self.id)*10 + len(self.parmConfObjList) + 1
296
290
297 def updateId(self, new_id):
291 def updateId(self, new_id):
298
292
299 self.id = str(new_id)
293 self.id = str(new_id)
300
294
301 n = 1
295 n = 1
302 for parmObj in self.parmConfObjList:
296 for parmObj in self.parmConfObjList:
303
297
304 idParm = str(int(new_id)*10 + n)
298 idParm = str(int(new_id)*10 + n)
305 parmObj.updateId(idParm)
299 parmObj.updateId(idParm)
306
300
307 n += 1
301 n += 1
308
302
309 def getElementName(self):
303 def getElementName(self):
310
304
311 return self.ELEMENTNAME
305 return self.ELEMENTNAME
312
306
313 def getParameterObjList(self):
307 def getParameterObjList(self):
314
308
315 return self.parmConfObjList
309 return self.parmConfObjList
316
310
317 def getParameterObj(self, parameterName):
311 def getParameterObj(self, parameterName):
318
312
319 for parmConfObj in self.parmConfObjList:
313 for parmConfObj in self.parmConfObjList:
320
314
321 if parmConfObj.name != parameterName:
315 if parmConfObj.name != parameterName:
322 continue
316 continue
323
317
324 return parmConfObj
318 return parmConfObj
325
319
326 return None
320 return None
327
321
328 def getParameterObjfromValue(self, parameterValue):
322 def getParameterObjfromValue(self, parameterValue):
329
323
330 for parmConfObj in self.parmConfObjList:
324 for parmConfObj in self.parmConfObjList:
331
325
332 if parmConfObj.getValue() != parameterValue:
326 if parmConfObj.getValue() != parameterValue:
333 continue
327 continue
334
328
335 return parmConfObj.getValue()
329 return parmConfObj.getValue()
336
330
337 return None
331 return None
338
332
339 def getParameterValue(self, parameterName):
333 def getParameterValue(self, parameterName):
340
334
341 parameterObj = self.getParameterObj(parameterName)
335 parameterObj = self.getParameterObj(parameterName)
342
336
343 # if not parameterObj:
337 # if not parameterObj:
344 # return None
338 # return None
345
339
346 value = parameterObj.getValue()
340 value = parameterObj.getValue()
347
341
348 return value
342 return value
349
343
350
344
351 def getKwargs(self):
345 def getKwargs(self):
352
346
353 kwargs = {}
347 kwargs = {}
354
348
355 for parmConfObj in self.parmConfObjList:
349 for parmConfObj in self.parmConfObjList:
356 if self.name == 'run' and parmConfObj.name == 'datatype':
350 if self.name == 'run' and parmConfObj.name == 'datatype':
357 continue
351 continue
358
352
359 kwargs[parmConfObj.name] = parmConfObj.getValue()
353 kwargs[parmConfObj.name] = parmConfObj.getValue()
360
354
361 return kwargs
355 return kwargs
362
356
363 def setup(self, id, name, priority, type):
357 def setup(self, id, name, priority, type):
364
358
365 self.id = str(id)
359 self.id = str(id)
366 self.name = name
360 self.name = name
367 self.type = type
361 self.type = type
368 self.priority = priority
362 self.priority = priority
369
363
370 self.parmConfObjList = []
364 self.parmConfObjList = []
371
365
372 def removeParameters(self):
366 def removeParameters(self):
373
367
374 for obj in self.parmConfObjList:
368 for obj in self.parmConfObjList:
375 del obj
369 del obj
376
370
377 self.parmConfObjList = []
371 self.parmConfObjList = []
378
372
379 def addParameter(self, name, value, format='str'):
373 def addParameter(self, name, value, format='str'):
380
374
381 id = self.__getNewId()
375 id = self.__getNewId()
382
376
383 parmConfObj = ParameterConf()
377 parmConfObj = ParameterConf()
384 if not parmConfObj.setup(id, name, value, format):
378 if not parmConfObj.setup(id, name, value, format):
385 return None
379 return None
386
380
387 self.parmConfObjList.append(parmConfObj)
381 self.parmConfObjList.append(parmConfObj)
388
382
389 return parmConfObj
383 return parmConfObj
390
384
391 def changeParameter(self, name, value, format='str'):
385 def changeParameter(self, name, value, format='str'):
392
386
393 parmConfObj = self.getParameterObj(name)
387 parmConfObj = self.getParameterObj(name)
394 parmConfObj.update(name, value, format)
388 parmConfObj.update(name, value, format)
395
389
396 return parmConfObj
390 return parmConfObj
397
391
398 def makeXml(self, procUnitElement):
392 def makeXml(self, procUnitElement):
399
393
400 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
394 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
401 opElement.set('id', str(self.id))
395 opElement.set('id', str(self.id))
402 opElement.set('name', self.name)
396 opElement.set('name', self.name)
403 opElement.set('type', self.type)
397 opElement.set('type', self.type)
404 opElement.set('priority', str(self.priority))
398 opElement.set('priority', str(self.priority))
405
399
406 for parmConfObj in self.parmConfObjList:
400 for parmConfObj in self.parmConfObjList:
407 parmConfObj.makeXml(opElement)
401 parmConfObj.makeXml(opElement)
408
402
409 def readXml(self, opElement):
403 def readXml(self, opElement):
410
404
411 self.id = opElement.get('id')
405 self.id = opElement.get('id')
412 self.name = opElement.get('name')
406 self.name = opElement.get('name')
413 self.type = opElement.get('type')
407 self.type = opElement.get('type')
414 self.priority = opElement.get('priority')
408 self.priority = opElement.get('priority')
415
409
416 #Compatible with old signal chain version
410 #Compatible with old signal chain version
417 #Use of 'run' method instead 'init'
411 #Use of 'run' method instead 'init'
418 if self.type == 'self' and self.name == 'init':
412 if self.type == 'self' and self.name == 'init':
419 self.name = 'run'
413 self.name = 'run'
420
414
421 self.parmConfObjList = []
415 self.parmConfObjList = []
422
416
423 parmElementList = opElement.iter(ParameterConf().getElementName())
417 parmElementList = opElement.iter(ParameterConf().getElementName())
424
418
425 for parmElement in parmElementList:
419 for parmElement in parmElementList:
426 parmConfObj = ParameterConf()
420 parmConfObj = ParameterConf()
427 parmConfObj.readXml(parmElement)
421 parmConfObj.readXml(parmElement)
428
422
429 #Compatible with old signal chain version
423 #Compatible with old signal chain version
430 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
424 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
431 if self.type != 'self' and self.name == 'Plot':
425 if self.type != 'self' and self.name == 'Plot':
432 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
426 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
433 self.name = parmConfObj.value
427 self.name = parmConfObj.value
434 continue
428 continue
435
429
436 self.parmConfObjList.append(parmConfObj)
430 self.parmConfObjList.append(parmConfObj)
437
431
438 def printattr(self):
432 def printattr(self):
439
433
440 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
434 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
441 self.id,
435 self.id,
442 self.name,
436 self.name,
443 self.type,
437 self.type,
444 self.priority)
438 self.priority)
445
439
446 for parmConfObj in self.parmConfObjList:
440 for parmConfObj in self.parmConfObjList:
447 parmConfObj.printattr()
441 parmConfObj.printattr()
448
442
449 def createObject(self, plotter_queue=None):
443 def createObject(self, plotter_queue=None):
450
444
451
445
452 if self.type == 'self':
446 if self.type == 'self':
453 raise ValueError, "This operation type cannot be created"
447 raise ValueError, "This operation type cannot be created"
454
448
455 if self.type == 'plotter':
449 if self.type == 'plotter':
456 #Plotter(plotter_name)
450 #Plotter(plotter_name)
457 if not plotter_queue:
451 if not plotter_queue:
458 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
452 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
459
453
460 opObj = Plotter(self.name, plotter_queue)
454 opObj = Plotter(self.name, plotter_queue)
461
455
462 if self.type == 'external' or self.type == 'other':
456 if self.type == 'external' or self.type == 'other':
463
457
464 className = eval(self.name)
458 className = eval(self.name)
465 kwargs = self.getKwargs()
459 kwargs = self.getKwargs()
466
460
467 opObj = className(**kwargs)
461 opObj = className(**kwargs)
468
462
469 return opObj
463 return opObj
470
464
471
465
472 class ProcUnitConf():
466 class ProcUnitConf():
473
467
474 id = None
468 id = None
475 name = None
469 name = None
476 datatype = None
470 datatype = None
477 inputId = None
471 inputId = None
478 parentId = None
472 parentId = None
479
473
480 opConfObjList = []
474 opConfObjList = []
481
475
482 procUnitObj = None
476 procUnitObj = None
483 opObjList = []
477 opObjList = []
484
478
485 ELEMENTNAME = 'ProcUnit'
479 ELEMENTNAME = 'ProcUnit'
486
480
487 def __init__(self):
481 def __init__(self):
488
482
489 self.id = None
483 self.id = None
490 self.datatype = None
484 self.datatype = None
491 self.name = None
485 self.name = None
492 self.inputId = None
486 self.inputId = None
493
487
494 self.opConfObjList = []
488 self.opConfObjList = []
495
489
496 self.procUnitObj = None
490 self.procUnitObj = None
497 self.opObjDict = {}
491 self.opObjDict = {}
498
492
499 def __getPriority(self):
493 def __getPriority(self):
500
494
501 return len(self.opConfObjList)+1
495 return len(self.opConfObjList)+1
502
496
503 def __getNewId(self):
497 def __getNewId(self):
504
498
505 return int(self.id)*10 + len(self.opConfObjList) + 1
499 return int(self.id)*10 + len(self.opConfObjList) + 1
506
500
507 def getElementName(self):
501 def getElementName(self):
508
502
509 return self.ELEMENTNAME
503 return self.ELEMENTNAME
510
504
511 def getId(self):
505 def getId(self):
512
506
513 return self.id
507 return self.id
514
508
515 def updateId(self, new_id, parentId=parentId):
509 def updateId(self, new_id, parentId=parentId):
516
510
517
511
518 new_id = int(parentId)*10 + (int(self.id) % 10)
512 new_id = int(parentId)*10 + (int(self.id) % 10)
519 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
513 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
520
514
521 #If this proc unit has not inputs
515 #If this proc unit has not inputs
522 if self.inputId == '0':
516 if self.inputId == '0':
523 new_inputId = 0
517 new_inputId = 0
524
518
525 n = 1
519 n = 1
526 for opConfObj in self.opConfObjList:
520 for opConfObj in self.opConfObjList:
527
521
528 idOp = str(int(new_id)*10 + n)
522 idOp = str(int(new_id)*10 + n)
529 opConfObj.updateId(idOp)
523 opConfObj.updateId(idOp)
530
524
531 n += 1
525 n += 1
532
526
533 self.parentId = str(parentId)
527 self.parentId = str(parentId)
534 self.id = str(new_id)
528 self.id = str(new_id)
535 self.inputId = str(new_inputId)
529 self.inputId = str(new_inputId)
536
530
537
531
538 def getInputId(self):
532 def getInputId(self):
539
533
540 return self.inputId
534 return self.inputId
541
535
542 def getOperationObjList(self):
536 def getOperationObjList(self):
543
537
544 return self.opConfObjList
538 return self.opConfObjList
545
539
546 def getOperationObj(self, name=None):
540 def getOperationObj(self, name=None):
547
541
548 for opConfObj in self.opConfObjList:
542 for opConfObj in self.opConfObjList:
549
543
550 if opConfObj.name != name:
544 if opConfObj.name != name:
551 continue
545 continue
552
546
553 return opConfObj
547 return opConfObj
554
548
555 return None
549 return None
556
550
557 def getOpObjfromParamValue(self, value=None):
551 def getOpObjfromParamValue(self, value=None):
558
552
559 for opConfObj in self.opConfObjList:
553 for opConfObj in self.opConfObjList:
560 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
554 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
561 continue
555 continue
562 return opConfObj
556 return opConfObj
563 return None
557 return None
564
558
565 def getProcUnitObj(self):
559 def getProcUnitObj(self):
566
560
567 return self.procUnitObj
561 return self.procUnitObj
568
562
569 def setup(self, id, name, datatype, inputId, parentId=None):
563 def setup(self, id, name, datatype, inputId, parentId=None):
570
564
571 #Compatible with old signal chain version
565 #Compatible with old signal chain version
572 if datatype==None and name==None:
566 if datatype==None and name==None:
573 raise ValueError, "datatype or name should be defined"
567 raise ValueError, "datatype or name should be defined"
574
568
575 if name==None:
569 if name==None:
576 if 'Proc' in datatype:
570 if 'Proc' in datatype:
577 name = datatype
571 name = datatype
578 else:
572 else:
579 name = '%sProc' %(datatype)
573 name = '%sProc' %(datatype)
580
574
581 if datatype==None:
575 if datatype==None:
582 datatype = name.replace('Proc','')
576 datatype = name.replace('Proc','')
583
577
584 self.id = str(id)
578 self.id = str(id)
585 self.name = name
579 self.name = name
586 self.datatype = datatype
580 self.datatype = datatype
587 self.inputId = inputId
581 self.inputId = inputId
588 self.parentId = parentId
582 self.parentId = parentId
589
583
590 self.opConfObjList = []
584 self.opConfObjList = []
591
585
592 self.addOperation(name='run', optype='self')
586 self.addOperation(name='run', optype='self')
593
587
594 def removeOperations(self):
588 def removeOperations(self):
595
589
596 for obj in self.opConfObjList:
590 for obj in self.opConfObjList:
597 del obj
591 del obj
598
592
599 self.opConfObjList = []
593 self.opConfObjList = []
600 self.addOperation(name='run')
594 self.addOperation(name='run')
601
595
602 def addParameter(self, **kwargs):
596 def addParameter(self, **kwargs):
603 '''
597 '''
604 Add parameters to "run" operation
598 Add parameters to "run" operation
605 '''
599 '''
606 opObj = self.opConfObjList[0]
600 opObj = self.opConfObjList[0]
607
601
608 opObj.addParameter(**kwargs)
602 opObj.addParameter(**kwargs)
609
603
610 return opObj
604 return opObj
611
605
612 def addOperation(self, name, optype='self'):
606 def addOperation(self, name, optype='self'):
613
607
614 id = self.__getNewId()
608 id = self.__getNewId()
615 priority = self.__getPriority()
609 priority = self.__getPriority()
616
610
617 opConfObj = OperationConf()
611 opConfObj = OperationConf()
618 opConfObj.setup(id, name=name, priority=priority, type=optype)
612 opConfObj.setup(id, name=name, priority=priority, type=optype)
619
613
620 self.opConfObjList.append(opConfObj)
614 self.opConfObjList.append(opConfObj)
621
615
622 return opConfObj
616 return opConfObj
623
617
624 def makeXml(self, projectElement):
618 def makeXml(self, projectElement):
625
619
626 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
620 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
627 procUnitElement.set('id', str(self.id))
621 procUnitElement.set('id', str(self.id))
628 procUnitElement.set('name', self.name)
622 procUnitElement.set('name', self.name)
629 procUnitElement.set('datatype', self.datatype)
623 procUnitElement.set('datatype', self.datatype)
630 procUnitElement.set('inputId', str(self.inputId))
624 procUnitElement.set('inputId', str(self.inputId))
631
625
632 for opConfObj in self.opConfObjList:
626 for opConfObj in self.opConfObjList:
633 opConfObj.makeXml(procUnitElement)
627 opConfObj.makeXml(procUnitElement)
634
628
635 def readXml(self, upElement):
629 def readXml(self, upElement):
636
630
637 self.id = upElement.get('id')
631 self.id = upElement.get('id')
638 self.name = upElement.get('name')
632 self.name = upElement.get('name')
639 self.datatype = upElement.get('datatype')
633 self.datatype = upElement.get('datatype')
640 self.inputId = upElement.get('inputId')
634 self.inputId = upElement.get('inputId')
641
635
642 if self.ELEMENTNAME == "ReadUnit":
636 if self.ELEMENTNAME == "ReadUnit":
643 self.datatype = self.datatype.replace("Reader", "")
637 self.datatype = self.datatype.replace("Reader", "")
644
638
645 if self.ELEMENTNAME == "ProcUnit":
639 if self.ELEMENTNAME == "ProcUnit":
646 self.datatype = self.datatype.replace("Proc", "")
640 self.datatype = self.datatype.replace("Proc", "")
647
641
648 if self.inputId == 'None':
642 if self.inputId == 'None':
649 self.inputId = '0'
643 self.inputId = '0'
650
644
651 self.opConfObjList = []
645 self.opConfObjList = []
652
646
653 opElementList = upElement.iter(OperationConf().getElementName())
647 opElementList = upElement.iter(OperationConf().getElementName())
654
648
655 for opElement in opElementList:
649 for opElement in opElementList:
656 opConfObj = OperationConf()
650 opConfObj = OperationConf()
657 opConfObj.readXml(opElement)
651 opConfObj.readXml(opElement)
658 self.opConfObjList.append(opConfObj)
652 self.opConfObjList.append(opConfObj)
659
653
660 def printattr(self):
654 def printattr(self):
661
655
662 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
656 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
663 self.id,
657 self.id,
664 self.name,
658 self.name,
665 self.datatype,
659 self.datatype,
666 self.inputId)
660 self.inputId)
667
661
668 for opConfObj in self.opConfObjList:
662 for opConfObj in self.opConfObjList:
669 opConfObj.printattr()
663 opConfObj.printattr()
670
664
671
665
672 def getKwargs(self):
666 def getKwargs(self):
673
667
674 opObj = self.opConfObjList[0]
668 opObj = self.opConfObjList[0]
675 kwargs = opObj.getKwargs()
669 kwargs = opObj.getKwargs()
676
670
677 return kwargs
671 return kwargs
678
672
679 def createObjects(self, plotter_queue=None):
673 def createObjects(self, plotter_queue=None):
680
674
681 className = eval(self.name)
675 className = eval(self.name)
682 kwargs = self.getKwargs()
676 kwargs = self.getKwargs()
683 procUnitObj = className(**kwargs)
677 procUnitObj = className(**kwargs)
684
678
685 for opConfObj in self.opConfObjList:
679 for opConfObj in self.opConfObjList:
686
680
687 if opConfObj.type=='self' and self.name=='run':
681 if opConfObj.type=='self' and self.name=='run':
688 continue
682 continue
689 elif opConfObj.type=='self':
683 elif opConfObj.type=='self':
690 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
684 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
691 continue
685 continue
692
686
693 opObj = opConfObj.createObject(plotter_queue)
687 opObj = opConfObj.createObject(plotter_queue)
694
688
695 self.opObjDict[opConfObj.id] = opObj
689 self.opObjDict[opConfObj.id] = opObj
696
690
697 procUnitObj.addOperation(opObj, opConfObj.id)
691 procUnitObj.addOperation(opObj, opConfObj.id)
698
692
699 self.procUnitObj = procUnitObj
693 self.procUnitObj = procUnitObj
700
694
701 return procUnitObj
695 return procUnitObj
702
696
703 def run(self):
697 def run(self):
704
698
705 is_ok = False
699 is_ok = False
706
700
707 for opConfObj in self.opConfObjList:
701 for opConfObj in self.opConfObjList:
708
702
709 kwargs = {}
703 kwargs = {}
710 for parmConfObj in opConfObj.getParameterObjList():
704 for parmConfObj in opConfObj.getParameterObjList():
711 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
705 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
712 continue
706 continue
713
707
714 kwargs[parmConfObj.name] = parmConfObj.getValue()
708 kwargs[parmConfObj.name] = parmConfObj.getValue()
715
709
716 #ini = time.time()
710 #ini = time.time()
717
711
718 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
712 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
719 sts = self.procUnitObj.call(opType = opConfObj.type,
713 sts = self.procUnitObj.call(opType = opConfObj.type,
720 opName = opConfObj.name,
714 opName = opConfObj.name,
721 opId = opConfObj.id)
715 opId = opConfObj.id)
722
716
723 # total_time = time.time() - ini
717 # total_time = time.time() - ini
724 #
718 #
725 # if total_time > 0.002:
719 # if total_time > 0.002:
726 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
720 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
727
721
728 is_ok = is_ok or sts
722 is_ok = is_ok or sts
729
723
730 return is_ok
724 return is_ok
731
725
732 def close(self):
726 def close(self):
733
727
734 for opConfObj in self.opConfObjList:
728 for opConfObj in self.opConfObjList:
735 if opConfObj.type == 'self':
729 if opConfObj.type == 'self':
736 continue
730 continue
737
731
738 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
732 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
739 opObj.close()
733 opObj.close()
740
734
741 self.procUnitObj.close()
735 self.procUnitObj.close()
742
736
743 return
737 return
744
738
745 class ReadUnitConf(ProcUnitConf):
739 class ReadUnitConf(ProcUnitConf):
746
740
747 path = None
741 path = None
748 startDate = None
742 startDate = None
749 endDate = None
743 endDate = None
750 startTime = None
744 startTime = None
751 endTime = None
745 endTime = None
752
746
753 ELEMENTNAME = 'ReadUnit'
747 ELEMENTNAME = 'ReadUnit'
754
748
755 def __init__(self):
749 def __init__(self):
756
750
757 self.id = None
751 self.id = None
758 self.datatype = None
752 self.datatype = None
759 self.name = None
753 self.name = None
760 self.inputId = None
754 self.inputId = None
761
755
762 self.parentId = None
756 self.parentId = None
763
757
764 self.opConfObjList = []
758 self.opConfObjList = []
765 self.opObjList = []
759 self.opObjList = []
766
760
767 def getElementName(self):
761 def getElementName(self):
768
762
769 return self.ELEMENTNAME
763 return self.ELEMENTNAME
770
764
771 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
765 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
772 endTime="", parentId=None, queue=None, server=None, **kwargs):
766 endTime="", parentId=None, queue=None, server=None, **kwargs):
773 #Compatible with old signal chain version
767 #Compatible with old signal chain version
774 if datatype==None and name==None:
768 if datatype==None and name==None:
775 raise ValueError, "datatype or name should be defined"
769 raise ValueError, "datatype or name should be defined"
776
770
777 if name==None:
771 if name==None:
778 if 'Reader' in datatype:
772 if 'Reader' in datatype:
779 name = datatype
773 name = datatype
780 else:
774 else:
781 name = '%sReader' %(datatype)
775 name = '%sReader' %(datatype)
782 if datatype==None:
776 if datatype==None:
783 datatype = name.replace('Reader','')
777 datatype = name.replace('Reader','')
784
778
785 self.id = id
779 self.id = id
786 self.name = name
780 self.name = name
787 self.datatype = datatype
781 self.datatype = datatype
788 if path != '':
782 if path != '':
789 self.path = os.path.abspath(path)
783 self.path = os.path.abspath(path)
790 self.startDate = startDate
784 self.startDate = startDate
791 self.endDate = endDate
785 self.endDate = endDate
792 self.startTime = startTime
786 self.startTime = startTime
793 self.endTime = endTime
787 self.endTime = endTime
794
788
795 self.inputId = '0'
789 self.inputId = '0'
796 self.parentId = parentId
790 self.parentId = parentId
797 self.queue = queue
791 self.queue = queue
798 self.server = server
792 self.server = server
799 self.addRunOperation(**kwargs)
793 self.addRunOperation(**kwargs)
800
794
801 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
795 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
802
796
803 #Compatible with old signal chain version
797 #Compatible with old signal chain version
804 if datatype==None and name==None:
798 if datatype==None and name==None:
805 raise ValueError, "datatype or name should be defined"
799 raise ValueError, "datatype or name should be defined"
806
800
807 if name==None:
801 if name==None:
808 if 'Reader' in datatype:
802 if 'Reader' in datatype:
809 name = datatype
803 name = datatype
810 else:
804 else:
811 name = '%sReader' %(datatype)
805 name = '%sReader' %(datatype)
812
806
813 if datatype==None:
807 if datatype==None:
814 datatype = name.replace('Reader','')
808 datatype = name.replace('Reader','')
815
809
816 self.datatype = datatype
810 self.datatype = datatype
817 self.name = name
811 self.name = name
818 self.path = path
812 self.path = path
819 self.startDate = startDate
813 self.startDate = startDate
820 self.endDate = endDate
814 self.endDate = endDate
821 self.startTime = startTime
815 self.startTime = startTime
822 self.endTime = endTime
816 self.endTime = endTime
823
817
824 self.inputId = '0'
818 self.inputId = '0'
825 self.parentId = parentId
819 self.parentId = parentId
826
820
827 self.updateRunOperation(**kwargs)
821 self.updateRunOperation(**kwargs)
828
822
829 def removeOperations(self):
823 def removeOperations(self):
830
824
831 for obj in self.opConfObjList:
825 for obj in self.opConfObjList:
832 del obj
826 del obj
833
827
834 self.opConfObjList = []
828 self.opConfObjList = []
835
829
836 def addRunOperation(self, **kwargs):
830 def addRunOperation(self, **kwargs):
837
831
838 opObj = self.addOperation(name = 'run', optype = 'self')
832 opObj = self.addOperation(name = 'run', optype = 'self')
839
833
840 if self.server is None:
834 if self.server is None:
841 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
835 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
842 opObj.addParameter(name='path' , value=self.path, format='str')
836 opObj.addParameter(name='path' , value=self.path, format='str')
843 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
837 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
844 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
838 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
845 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
839 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
846 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
840 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
847 opObj.addParameter(name='queue' , value=self.queue, format='obj')
841 opObj.addParameter(name='queue' , value=self.queue, format='obj')
848 for key, value in kwargs.items():
842 for key, value in kwargs.items():
849 opObj.addParameter(name=key, value=value, format=type(value).__name__)
843 opObj.addParameter(name=key, value=value, format=type(value).__name__)
850 else:
844 else:
851 opObj.addParameter(name='server' , value=self.server, format='str')
845 opObj.addParameter(name='server' , value=self.server, format='str')
852
846
853
847
854 return opObj
848 return opObj
855
849
856 def updateRunOperation(self, **kwargs):
850 def updateRunOperation(self, **kwargs):
857
851
858 opObj = self.getOperationObj(name = 'run')
852 opObj = self.getOperationObj(name = 'run')
859 opObj.removeParameters()
853 opObj.removeParameters()
860
854
861 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
855 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
862 opObj.addParameter(name='path' , value=self.path, format='str')
856 opObj.addParameter(name='path' , value=self.path, format='str')
863 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
857 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
864 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
858 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
865 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
859 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
866 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
860 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
867
861
868 for key, value in kwargs.items():
862 for key, value in kwargs.items():
869 opObj.addParameter(name=key, value=value, format=type(value).__name__)
863 opObj.addParameter(name=key, value=value, format=type(value).__name__)
870
864
871 return opObj
865 return opObj
872
866
873 # def makeXml(self, projectElement):
867 # def makeXml(self, projectElement):
874 #
868 #
875 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
869 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
876 # procUnitElement.set('id', str(self.id))
870 # procUnitElement.set('id', str(self.id))
877 # procUnitElement.set('name', self.name)
871 # procUnitElement.set('name', self.name)
878 # procUnitElement.set('datatype', self.datatype)
872 # procUnitElement.set('datatype', self.datatype)
879 # procUnitElement.set('inputId', str(self.inputId))
873 # procUnitElement.set('inputId', str(self.inputId))
880 #
874 #
881 # for opConfObj in self.opConfObjList:
875 # for opConfObj in self.opConfObjList:
882 # opConfObj.makeXml(procUnitElement)
876 # opConfObj.makeXml(procUnitElement)
883
877
884 def readXml(self, upElement):
878 def readXml(self, upElement):
885
879
886 self.id = upElement.get('id')
880 self.id = upElement.get('id')
887 self.name = upElement.get('name')
881 self.name = upElement.get('name')
888 self.datatype = upElement.get('datatype')
882 self.datatype = upElement.get('datatype')
889 self.inputId = upElement.get('inputId')
883 self.inputId = upElement.get('inputId')
890
884
891 if self.ELEMENTNAME == "ReadUnit":
885 if self.ELEMENTNAME == "ReadUnit":
892 self.datatype = self.datatype.replace("Reader", "")
886 self.datatype = self.datatype.replace("Reader", "")
893
887
894 if self.inputId == 'None':
888 if self.inputId == 'None':
895 self.inputId = '0'
889 self.inputId = '0'
896
890
897 self.opConfObjList = []
891 self.opConfObjList = []
898
892
899 opElementList = upElement.iter(OperationConf().getElementName())
893 opElementList = upElement.iter(OperationConf().getElementName())
900
894
901 for opElement in opElementList:
895 for opElement in opElementList:
902 opConfObj = OperationConf()
896 opConfObj = OperationConf()
903 opConfObj.readXml(opElement)
897 opConfObj.readXml(opElement)
904 self.opConfObjList.append(opConfObj)
898 self.opConfObjList.append(opConfObj)
905
899
906 if opConfObj.name == 'run':
900 if opConfObj.name == 'run':
907 self.path = opConfObj.getParameterValue('path')
901 self.path = opConfObj.getParameterValue('path')
908 self.startDate = opConfObj.getParameterValue('startDate')
902 self.startDate = opConfObj.getParameterValue('startDate')
909 self.endDate = opConfObj.getParameterValue('endDate')
903 self.endDate = opConfObj.getParameterValue('endDate')
910 self.startTime = opConfObj.getParameterValue('startTime')
904 self.startTime = opConfObj.getParameterValue('startTime')
911 self.endTime = opConfObj.getParameterValue('endTime')
905 self.endTime = opConfObj.getParameterValue('endTime')
912
906
913 class Project(Process):
907 class Project(Process):
914 id = None
908 id = None
915 name = None
909 name = None
916 description = None
910 description = None
917 filename = None
911 filename = None
918
912
919 procUnitConfObjDict = None
913 procUnitConfObjDict = None
920
914
921 ELEMENTNAME = 'Project'
915 ELEMENTNAME = 'Project'
922
916
923 plotterQueue = None
917 plotterQueue = None
924
918
925 def __init__(self, plotter_queue=None, logfile=None):
919 def __init__(self, plotter_queue=None, logfile=None):
926 Process.__init__(self)
920 Process.__init__(self)
927 self.id = None
921 self.id = None
928 self.name = None
922 self.name = None
929 self.description = None
923 self.description = None
930 if logfile is not None:
924 if logfile is not None:
931 logToFile(logfile)
925 logToFile(logfile)
932 self.plotterQueue = plotter_queue
926 self.plotterQueue = plotter_queue
933
927
934 self.procUnitConfObjDict = {}
928 self.procUnitConfObjDict = {}
935
929
936 def __getNewId(self):
930 def __getNewId(self):
937
931
938 idList = self.procUnitConfObjDict.keys()
932 idList = self.procUnitConfObjDict.keys()
939
933
940 id = int(self.id)*10
934 id = int(self.id)*10
941
935
942 while True:
936 while True:
943 id += 1
937 id += 1
944
938
945 if str(id) in idList:
939 if str(id) in idList:
946 continue
940 continue
947
941
948 break
942 break
949
943
950 return str(id)
944 return str(id)
951
945
952 def getElementName(self):
946 def getElementName(self):
953
947
954 return self.ELEMENTNAME
948 return self.ELEMENTNAME
955
949
956 def getId(self):
950 def getId(self):
957
951
958 return self.id
952 return self.id
959
953
960 def updateId(self, new_id):
954 def updateId(self, new_id):
961
955
962 self.id = str(new_id)
956 self.id = str(new_id)
963
957
964 keyList = self.procUnitConfObjDict.keys()
958 keyList = self.procUnitConfObjDict.keys()
965 keyList.sort()
959 keyList.sort()
966
960
967 n = 1
961 n = 1
968 newProcUnitConfObjDict = {}
962 newProcUnitConfObjDict = {}
969
963
970 for procKey in keyList:
964 for procKey in keyList:
971
965
972 procUnitConfObj = self.procUnitConfObjDict[procKey]
966 procUnitConfObj = self.procUnitConfObjDict[procKey]
973 idProcUnit = str(int(self.id)*10 + n)
967 idProcUnit = str(int(self.id)*10 + n)
974 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
968 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
975
969
976 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
970 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
977 n += 1
971 n += 1
978
972
979 self.procUnitConfObjDict = newProcUnitConfObjDict
973 self.procUnitConfObjDict = newProcUnitConfObjDict
980
974
981 def setup(self, id, name, description):
975 def setup(self, id, name, description):
982
976
983 self.id = str(id)
977 self.id = str(id)
984 self.name = name
978 self.name = name
985 self.description = description
979 self.description = description
986
980
987 def update(self, name, description):
981 def update(self, name, description):
988
982
989 self.name = name
983 self.name = name
990 self.description = description
984 self.description = description
991
985
992 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
986 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
993 if id is None:
987 if id is None:
994 idReadUnit = self.__getNewId()
988 idReadUnit = self.__getNewId()
995 else:
989 else:
996 idReadUnit = str(id)
990 idReadUnit = str(id)
997
991
998 readUnitConfObj = ReadUnitConf()
992 readUnitConfObj = ReadUnitConf()
999 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
993 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
1000
994
1001 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
995 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1002
996
1003 return readUnitConfObj
997 return readUnitConfObj
1004
998
1005 def addProcUnit(self, inputId='0', datatype=None, name=None):
999 def addProcUnit(self, inputId='0', datatype=None, name=None):
1006
1000
1007 idProcUnit = self.__getNewId()
1001 idProcUnit = self.__getNewId()
1008
1002
1009 procUnitConfObj = ProcUnitConf()
1003 procUnitConfObj = ProcUnitConf()
1010 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1004 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1011
1005
1012 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1006 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1013
1007
1014 return procUnitConfObj
1008 return procUnitConfObj
1015
1009
1016 def removeProcUnit(self, id):
1010 def removeProcUnit(self, id):
1017
1011
1018 if id in self.procUnitConfObjDict.keys():
1012 if id in self.procUnitConfObjDict.keys():
1019 self.procUnitConfObjDict.pop(id)
1013 self.procUnitConfObjDict.pop(id)
1020
1014
1021 def getReadUnitId(self):
1015 def getReadUnitId(self):
1022
1016
1023 readUnitConfObj = self.getReadUnitObj()
1017 readUnitConfObj = self.getReadUnitObj()
1024
1018
1025 return readUnitConfObj.id
1019 return readUnitConfObj.id
1026
1020
1027 def getReadUnitObj(self):
1021 def getReadUnitObj(self):
1028
1022
1029 for obj in self.procUnitConfObjDict.values():
1023 for obj in self.procUnitConfObjDict.values():
1030 if obj.getElementName() == "ReadUnit":
1024 if obj.getElementName() == "ReadUnit":
1031 return obj
1025 return obj
1032
1026
1033 return None
1027 return None
1034
1028
1035 def getProcUnitObj(self, id=None, name=None):
1029 def getProcUnitObj(self, id=None, name=None):
1036
1030
1037 if id != None:
1031 if id != None:
1038 return self.procUnitConfObjDict[id]
1032 return self.procUnitConfObjDict[id]
1039
1033
1040 if name != None:
1034 if name != None:
1041 return self.getProcUnitObjByName(name)
1035 return self.getProcUnitObjByName(name)
1042
1036
1043 return None
1037 return None
1044
1038
1045 def getProcUnitObjByName(self, name):
1039 def getProcUnitObjByName(self, name):
1046
1040
1047 for obj in self.procUnitConfObjDict.values():
1041 for obj in self.procUnitConfObjDict.values():
1048 if obj.name == name:
1042 if obj.name == name:
1049 return obj
1043 return obj
1050
1044
1051 return None
1045 return None
1052
1046
1053 def procUnitItems(self):
1047 def procUnitItems(self):
1054
1048
1055 return self.procUnitConfObjDict.items()
1049 return self.procUnitConfObjDict.items()
1056
1050
1057 def makeXml(self):
1051 def makeXml(self):
1058
1052
1059 projectElement = Element('Project')
1053 projectElement = Element('Project')
1060 projectElement.set('id', str(self.id))
1054 projectElement.set('id', str(self.id))
1061 projectElement.set('name', self.name)
1055 projectElement.set('name', self.name)
1062 projectElement.set('description', self.description)
1056 projectElement.set('description', self.description)
1063
1057
1064 for procUnitConfObj in self.procUnitConfObjDict.values():
1058 for procUnitConfObj in self.procUnitConfObjDict.values():
1065 procUnitConfObj.makeXml(projectElement)
1059 procUnitConfObj.makeXml(projectElement)
1066
1060
1067 self.projectElement = projectElement
1061 self.projectElement = projectElement
1068
1062
1069 def writeXml(self, filename=None):
1063 def writeXml(self, filename=None):
1070
1064
1071 if filename == None:
1065 if filename == None:
1072 if self.filename:
1066 if self.filename:
1073 filename = self.filename
1067 filename = self.filename
1074 else:
1068 else:
1075 filename = "schain.xml"
1069 filename = "schain.xml"
1076
1070
1077 if not filename:
1071 if not filename:
1078 print "filename has not been defined. Use setFilename(filename) for do it."
1072 print "filename has not been defined. Use setFilename(filename) for do it."
1079 return 0
1073 return 0
1080
1074
1081 abs_file = os.path.abspath(filename)
1075 abs_file = os.path.abspath(filename)
1082
1076
1083 if not os.access(os.path.dirname(abs_file), os.W_OK):
1077 if not os.access(os.path.dirname(abs_file), os.W_OK):
1084 print "No write permission on %s" %os.path.dirname(abs_file)
1078 print "No write permission on %s" %os.path.dirname(abs_file)
1085 return 0
1079 return 0
1086
1080
1087 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1081 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1088 print "File %s already exists and it could not be overwriten" %abs_file
1082 print "File %s already exists and it could not be overwriten" %abs_file
1089 return 0
1083 return 0
1090
1084
1091 self.makeXml()
1085 self.makeXml()
1092
1086
1093 ElementTree(self.projectElement).write(abs_file, method='xml')
1087 ElementTree(self.projectElement).write(abs_file, method='xml')
1094
1088
1095 self.filename = abs_file
1089 self.filename = abs_file
1096
1090
1097 return 1
1091 return 1
1098
1092
1099 def readXml(self, filename = None):
1093 def readXml(self, filename = None):
1100
1094
1101 if not filename:
1095 if not filename:
1102 print "filename is not defined"
1096 print "filename is not defined"
1103 return 0
1097 return 0
1104
1098
1105 abs_file = os.path.abspath(filename)
1099 abs_file = os.path.abspath(filename)
1106
1100
1107 if not os.path.isfile(abs_file):
1101 if not os.path.isfile(abs_file):
1108 print "%s file does not exist" %abs_file
1102 print "%s file does not exist" %abs_file
1109 return 0
1103 return 0
1110
1104
1111 self.projectElement = None
1105 self.projectElement = None
1112 self.procUnitConfObjDict = {}
1106 self.procUnitConfObjDict = {}
1113
1107
1114 try:
1108 try:
1115 self.projectElement = ElementTree().parse(abs_file)
1109 self.projectElement = ElementTree().parse(abs_file)
1116 except:
1110 except:
1117 print "Error reading %s, verify file format" %filename
1111 print "Error reading %s, verify file format" %filename
1118 return 0
1112 return 0
1119
1113
1120 self.project = self.projectElement.tag
1114 self.project = self.projectElement.tag
1121
1115
1122 self.id = self.projectElement.get('id')
1116 self.id = self.projectElement.get('id')
1123 self.name = self.projectElement.get('name')
1117 self.name = self.projectElement.get('name')
1124 self.description = self.projectElement.get('description')
1118 self.description = self.projectElement.get('description')
1125
1119
1126 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1120 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1127
1121
1128 for readUnitElement in readUnitElementList:
1122 for readUnitElement in readUnitElementList:
1129 readUnitConfObj = ReadUnitConf()
1123 readUnitConfObj = ReadUnitConf()
1130 readUnitConfObj.readXml(readUnitElement)
1124 readUnitConfObj.readXml(readUnitElement)
1131
1125
1132 if readUnitConfObj.parentId == None:
1126 if readUnitConfObj.parentId == None:
1133 readUnitConfObj.parentId = self.id
1127 readUnitConfObj.parentId = self.id
1134
1128
1135 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1129 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1136
1130
1137 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1131 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1138
1132
1139 for procUnitElement in procUnitElementList:
1133 for procUnitElement in procUnitElementList:
1140 procUnitConfObj = ProcUnitConf()
1134 procUnitConfObj = ProcUnitConf()
1141 procUnitConfObj.readXml(procUnitElement)
1135 procUnitConfObj.readXml(procUnitElement)
1142
1136
1143 if procUnitConfObj.parentId == None:
1137 if procUnitConfObj.parentId == None:
1144 procUnitConfObj.parentId = self.id
1138 procUnitConfObj.parentId = self.id
1145
1139
1146 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1140 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1147
1141
1148 self.filename = abs_file
1142 self.filename = abs_file
1149
1143
1150 return 1
1144 return 1
1151
1145
1152 def printattr(self):
1146 def printattr(self):
1153
1147
1154 print "Project[%s]: name = %s, description = %s" %(self.id,
1148 print "Project[%s]: name = %s, description = %s" %(self.id,
1155 self.name,
1149 self.name,
1156 self.description)
1150 self.description)
1157
1151
1158 for procUnitConfObj in self.procUnitConfObjDict.values():
1152 for procUnitConfObj in self.procUnitConfObjDict.values():
1159 procUnitConfObj.printattr()
1153 procUnitConfObj.printattr()
1160
1154
1161 def createObjects(self):
1155 def createObjects(self):
1162
1156
1163 for procUnitConfObj in self.procUnitConfObjDict.values():
1157 for procUnitConfObj in self.procUnitConfObjDict.values():
1164 procUnitConfObj.createObjects(self.plotterQueue)
1158 procUnitConfObj.createObjects(self.plotterQueue)
1165
1159
1166 def __connect(self, objIN, thisObj):
1160 def __connect(self, objIN, thisObj):
1167
1161
1168 thisObj.setInput(objIN.getOutputObj())
1162 thisObj.setInput(objIN.getOutputObj())
1169
1163
1170 def connectObjects(self):
1164 def connectObjects(self):
1171
1165
1172 for thisPUConfObj in self.procUnitConfObjDict.values():
1166 for thisPUConfObj in self.procUnitConfObjDict.values():
1173
1167
1174 inputId = thisPUConfObj.getInputId()
1168 inputId = thisPUConfObj.getInputId()
1175
1169
1176 if int(inputId) == 0:
1170 if int(inputId) == 0:
1177 continue
1171 continue
1178
1172
1179 #Get input object
1173 #Get input object
1180 puConfINObj = self.procUnitConfObjDict[inputId]
1174 puConfINObj = self.procUnitConfObjDict[inputId]
1181 puObjIN = puConfINObj.getProcUnitObj()
1175 puObjIN = puConfINObj.getProcUnitObj()
1182
1176
1183 #Get current object
1177 #Get current object
1184 thisPUObj = thisPUConfObj.getProcUnitObj()
1178 thisPUObj = thisPUConfObj.getProcUnitObj()
1185
1179
1186 self.__connect(puObjIN, thisPUObj)
1180 self.__connect(puObjIN, thisPUObj)
1187
1181
1188 def __handleError(self, procUnitConfObj, send_email=True):
1182 def __handleError(self, procUnitConfObj, send_email=True):
1189
1183
1190 import socket
1184 import socket
1191
1185
1192 err = traceback.format_exception(sys.exc_info()[0],
1186 err = traceback.format_exception(sys.exc_info()[0],
1193 sys.exc_info()[1],
1187 sys.exc_info()[1],
1194 sys.exc_info()[2])
1188 sys.exc_info()[2])
1195
1189
1196 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1190 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1197 print "***** %s" %err[-1]
1191 print "***** %s" %err[-1]
1198
1192
1199 message = "".join(err)
1193 message = "".join(err)
1200
1194
1201 sys.stderr.write(message)
1195 sys.stderr.write(message)
1202
1196
1203 if not send_email:
1197 if not send_email:
1204 return
1198 return
1205
1199
1206 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1200 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1207
1201
1208 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1202 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1209 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1203 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1210 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1204 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1211 subtitle += "Configuration file: %s\n" %self.filename
1205 subtitle += "Configuration file: %s\n" %self.filename
1212 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1206 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1213
1207
1214 readUnitConfObj = self.getReadUnitObj()
1208 readUnitConfObj = self.getReadUnitObj()
1215 if readUnitConfObj:
1209 if readUnitConfObj:
1216 subtitle += "\nInput parameters:\n"
1210 subtitle += "\nInput parameters:\n"
1217 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1211 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1218 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1212 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1219 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1213 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1220 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1214 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1221 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1215 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1222 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1216 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1223
1217
1224 adminObj = schainpy.admin.SchainNotify()
1218 adminObj = schainpy.admin.SchainNotify()
1225 adminObj.sendAlert(message=message,
1219 adminObj.sendAlert(message=message,
1226 subject=subject,
1220 subject=subject,
1227 subtitle=subtitle,
1221 subtitle=subtitle,
1228 filename=self.filename)
1222 filename=self.filename)
1229
1223
1230 def isPaused(self):
1224 def isPaused(self):
1231 return 0
1225 return 0
1232
1226
1233 def isStopped(self):
1227 def isStopped(self):
1234 return 0
1228 return 0
1235
1229
1236 def runController(self):
1230 def runController(self):
1237 """
1231 """
1238 returns 0 when this process has been stopped, 1 otherwise
1232 returns 0 when this process has been stopped, 1 otherwise
1239 """
1233 """
1240
1234
1241 if self.isPaused():
1235 if self.isPaused():
1242 print "Process suspended"
1236 print "Process suspended"
1243
1237
1244 while True:
1238 while True:
1245 sleep(0.1)
1239 sleep(0.1)
1246
1240
1247 if not self.isPaused():
1241 if not self.isPaused():
1248 break
1242 break
1249
1243
1250 if self.isStopped():
1244 if self.isStopped():
1251 break
1245 break
1252
1246
1253 print "Process reinitialized"
1247 print "Process reinitialized"
1254
1248
1255 if self.isStopped():
1249 if self.isStopped():
1256 print "Process stopped"
1250 print "Process stopped"
1257 return 0
1251 return 0
1258
1252
1259 return 1
1253 return 1
1260
1254
1261 def setFilename(self, filename):
1255 def setFilename(self, filename):
1262
1256
1263 self.filename = filename
1257 self.filename = filename
1264
1258
1265 def setPlotterQueue(self, plotter_queue):
1259 def setPlotterQueue(self, plotter_queue):
1266
1260
1267 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1261 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1268
1262
1269 def getPlotterQueue(self):
1263 def getPlotterQueue(self):
1270
1264
1271 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1272
1266
1273 def useExternalPlotter(self):
1267 def useExternalPlotter(self):
1274
1268
1275 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1276
1270
1277
1271
1278 def run(self, filename=None):
1272 def run(self, filename=None):
1279
1273
1280 # self.writeXml(filename)
1274 # self.writeXml(filename)
1281 self.createObjects()
1275 self.createObjects()
1282 self.connectObjects()
1276 self.connectObjects()
1283
1277
1284 print
1278 print
1285 print "*"*60
1279 print "*"*60
1286 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1280 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1287 print "*"*60
1281 print "*"*60
1288 print
1282 print
1289
1283
1290 keyList = self.procUnitConfObjDict.keys()
1284 keyList = self.procUnitConfObjDict.keys()
1291 keyList.sort()
1285 keyList.sort()
1292
1286
1293 while(True):
1287 while(True):
1294
1288
1295 is_ok = False
1289 is_ok = False
1296
1290
1297 for procKey in keyList:
1291 for procKey in keyList:
1298 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1292 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1299
1293
1300 procUnitConfObj = self.procUnitConfObjDict[procKey]
1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1301
1295
1302 try:
1296 try:
1303 sts = procUnitConfObj.run()
1297 sts = procUnitConfObj.run()
1304 is_ok = is_ok or sts
1298 is_ok = is_ok or sts
1305 except KeyboardInterrupt:
1299 except KeyboardInterrupt:
1306 is_ok = False
1300 is_ok = False
1307 break
1301 break
1308 except ValueError, e:
1302 except ValueError, e:
1309 sleep(0.5)
1303 sleep(0.5)
1310 self.__handleError(procUnitConfObj, send_email=True)
1304 self.__handleError(procUnitConfObj, send_email=True)
1311 is_ok = False
1305 is_ok = False
1312 break
1306 break
1313 except:
1307 except:
1314 sleep(0.5)
1308 sleep(0.5)
1315 self.__handleError(procUnitConfObj)
1309 self.__handleError(procUnitConfObj)
1316 is_ok = False
1310 is_ok = False
1317 break
1311 break
1318
1312
1319 #If every process unit finished so end process
1313 #If every process unit finished so end process
1320 if not(is_ok):
1314 if not(is_ok):
1321 # print "Every process unit have finished"
1315 # print "Every process unit have finished"
1322 break
1316 break
1323
1317
1324 if not self.runController():
1318 if not self.runController():
1325 break
1319 break
1326
1320
1327 #Closing every process
1321 #Closing every process
1328 for procKey in keyList:
1322 for procKey in keyList:
1329 procUnitConfObj = self.procUnitConfObjDict[procKey]
1323 procUnitConfObj = self.procUnitConfObjDict[procKey]
1330 procUnitConfObj.close()
1324 procUnitConfObj.close()
General Comments 0
You need to be logged in to leave comments. Login now