##// END OF EJS Templates
schain-jc
RSS
controller from master
José Chávez -
r1055:54f61554a2b5
parent child
Show More
Side by Side Unified
Context file:
r1055:54f61554a2b5 Loading diff...:
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (978 lines changed) Show them Hide them
@@ -1,16 +1,12
1 '''
1 '''
2 Created on September , 2012
2 Created on September , 2012
3 @author:
3 @author:
4 '''
4 '''
5
5
6 import sys
6 import sys
7 import ast
7 import ast
8 import datetime
8 import datetime
9 import traceback
9 import traceback
10 import math
11 import time
12 from multiprocessing import Process, Queue, cpu_count
13
14 import schainpy
10 import schainpy
15 import schainpy.admin
11 import schainpy.admin
16
12
@@ -27,141 +23,96 def prettify(elem):
27 reparsed = minidom.parseString(rough_string)
23 reparsed = minidom.parseString(rough_string)
28 return reparsed.toprettyxml(indent=" ")
24 return reparsed.toprettyxml(indent=" ")
29
25
30 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
31 skip = 0
32 cursor = 0
33 nFiles = None
34 processes = []
35 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
36 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
37 days = (dt2 - dt1).days
38
39 for day in range(days+1):
40 skip = 0
41 cursor = 0
42 q = Queue()
43 processes = []
44 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
45 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
46 firstProcess.start()
47 if by_day:
48 continue
49 nFiles = q.get()
50 firstProcess.terminate()
51 skip = int(math.ceil(nFiles/nProcess))
52 while True:
53 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
54 processes[cursor].start()
55 if nFiles < cursor*skip:
56 break
57 cursor += 1
58
59 def beforeExit(exctype, value, trace):
60 for process in processes:
61 process.terminate()
62 process.join()
63 print traceback.print_tb(trace)
64
65 sys.excepthook = beforeExit
66
67 for process in processes:
68 process.join()
69 process.terminate()
70 time.sleep(3)
71
72 class ParameterConf():
26 class ParameterConf():
73
27
74 id = None
28 id = None
75 name = None
29 name = None
76 value = None
30 value = None
77 format = None
31 format = None
78
32
79 __formated_value = None
33 __formated_value = None
80
34
81 ELEMENTNAME = 'Parameter'
35 ELEMENTNAME = 'Parameter'
82
36
83 def __init__(self):
37 def __init__(self):
84
38
85 self.format = 'str'
39 self.format = 'str'
86
40
87 def getElementName(self):
41 def getElementName(self):
88
42
89 return self.ELEMENTNAME
43 return self.ELEMENTNAME
90
44
91 def getValue(self):
45 def getValue(self):
92
46
93 value = self.value
47 value = self.value
94 format = self.format
48 format = self.format
95
49
96 if self.__formated_value != None:
50 if self.__formated_value != None:
97
51
98 return self.__formated_value
52 return self.__formated_value
99
53
100 if format == 'obj':
101 return value
102
103 if format == 'str':
54 if format == 'str':
104 self.__formated_value = str(value)
55 self.__formated_value = str(value)
105 return self.__formated_value
56 return self.__formated_value
106
57
107 if value == '':
58 if value == '':
108 raise ValueError, "%s: This parameter value is empty" %self.name
59 raise ValueError, "%s: This parameter value is empty" %self.name
109
60
110 if format == 'list':
61 if format == 'list':
111 strList = value.split(',')
62 strList = value.split(',')
112
63
113 self.__formated_value = strList
64 self.__formated_value = strList
114
65
115 return self.__formated_value
66 return self.__formated_value
116
67
117 if format == 'intlist':
68 if format == 'intlist':
118 """
69 """
119 Example:
70 Example:
120 value = (0,1,2)
71 value = (0,1,2)
121 """
72 """
122
73
123 new_value = ast.literal_eval(value)
74 new_value = ast.literal_eval(value)
124
75
125 if type(new_value) not in (tuple, list):
76 if type(new_value) not in (tuple, list):
126 new_value = [int(new_value)]
77 new_value = [int(new_value)]
127
78
128 self.__formated_value = new_value
79 self.__formated_value = new_value
129
80
130 return self.__formated_value
81 return self.__formated_value
131
82
132 if format == 'floatlist':
83 if format == 'floatlist':
133 """
84 """
134 Example:
85 Example:
135 value = (0.5, 1.4, 2.7)
86 value = (0.5, 1.4, 2.7)
136 """
87 """
137
88
138 new_value = ast.literal_eval(value)
89 new_value = ast.literal_eval(value)
139
90
140 if type(new_value) not in (tuple, list):
91 if type(new_value) not in (tuple, list):
141 new_value = [float(new_value)]
92 new_value = [float(new_value)]
142
93
143 self.__formated_value = new_value
94 self.__formated_value = new_value
144
95
145 return self.__formated_value
96 return self.__formated_value
146
97
147 if format == 'date':
98 if format == 'date':
148 strList = value.split('/')
99 strList = value.split('/')
149 intList = [int(x) for x in strList]
100 intList = [int(x) for x in strList]
150 date = datetime.date(intList[0], intList[1], intList[2])
101 date = datetime.date(intList[0], intList[1], intList[2])
151
102
152 self.__formated_value = date
103 self.__formated_value = date
153
104
154 return self.__formated_value
105 return self.__formated_value
155
106
156 if format == 'time':
107 if format == 'time':
157 strList = value.split(':')
108 strList = value.split(':')
158 intList = [int(x) for x in strList]
109 intList = [int(x) for x in strList]
159 time = datetime.time(intList[0], intList[1], intList[2])
110 time = datetime.time(intList[0], intList[1], intList[2])
160
111
161 self.__formated_value = time
112 self.__formated_value = time
162
113
163 return self.__formated_value
114 return self.__formated_value
164
115
165 if format == 'pairslist':
116 if format == 'pairslist':
166 """
117 """
167 Example:
118 Example:
@@ -169,483 +120,463 class ParameterConf():
169 """
120 """
170
121
171 new_value = ast.literal_eval(value)
122 new_value = ast.literal_eval(value)
172
123
173 if type(new_value) not in (tuple, list):
124 if type(new_value) not in (tuple, list):
174 raise ValueError, "%s has to be a tuple or list of pairs" %value
125 raise ValueError, "%s has to be a tuple or list of pairs" %value
175
126
176 if type(new_value[0]) not in (tuple, list):
127 if type(new_value[0]) not in (tuple, list):
177 if len(new_value) != 2:
128 if len(new_value) != 2:
178 raise ValueError, "%s has to be a tuple or list of pairs" %value
129 raise ValueError, "%s has to be a tuple or list of pairs" %value
179 new_value = [new_value]
130 new_value = [new_value]
180
131
181 for thisPair in new_value:
132 for thisPair in new_value:
182 if len(thisPair) != 2:
133 if len(thisPair) != 2:
183 raise ValueError, "%s has to be a tuple or list of pairs" %value
134 raise ValueError, "%s has to be a tuple or list of pairs" %value
184
135
185 self.__formated_value = new_value
136 self.__formated_value = new_value
186
137
187 return self.__formated_value
138 return self.__formated_value
188
139
189 if format == 'multilist':
140 if format == 'multilist':
190 """
141 """
191 Example:
142 Example:
192 value = (0,1,2),(3,4,5)
143 value = (0,1,2),(3,4,5)
193 """
144 """
194 multiList = ast.literal_eval(value)
145 multiList = ast.literal_eval(value)
195
146
196 if type(multiList[0]) == int:
147 if type(multiList[0]) == int:
197 multiList = ast.literal_eval("(" + value + ")")
148 multiList = ast.literal_eval("(" + value + ")")
198
149
199 self.__formated_value = multiList
150 self.__formated_value = multiList
200
151
201 return self.__formated_value
152 return self.__formated_value
202
153
203 if format == 'bool':
154 if format == 'bool':
204 value = int(value)
155 value = int(value)
205
156
206 if format == 'int':
157 if format == 'int':
207 value = float(value)
158 value = float(value)
208
159
209 format_func = eval(format)
160 format_func = eval(format)
210
161
211 self.__formated_value = format_func(value)
162 self.__formated_value = format_func(value)
212
163
213 return self.__formated_value
164 return self.__formated_value
214
165
215 def updateId(self, new_id):
166 def updateId(self, new_id):
216
167
217 self.id = str(new_id)
168 self.id = str(new_id)
218
169
219 def setup(self, id, name, value, format='str'):
170 def setup(self, id, name, value, format='str'):
171
220 self.id = str(id)
172 self.id = str(id)
221 self.name = name
173 self.name = name
222 if format == 'obj':
174 self.value = str(value)
223 self.value = value
224 else:
225 self.value = str(value)
226 self.format = str.lower(format)
175 self.format = str.lower(format)
227
176
228 self.getValue()
177 self.getValue()
229
178
230 return 1
179 return 1
231
180
232 def update(self, name, value, format='str'):
181 def update(self, name, value, format='str'):
233
182
234 self.name = name
183 self.name = name
235 self.value = str(value)
184 self.value = str(value)
236 self.format = format
185 self.format = format
237
186
238 def makeXml(self, opElement):
187 def makeXml(self, opElement):
239 if self.name not in ('queue',):
188
240 parmElement = SubElement(opElement, self.ELEMENTNAME)
189 parmElement = SubElement(opElement, self.ELEMENTNAME)
241 parmElement.set('id', str(self.id))
190 parmElement.set('id', str(self.id))
242 parmElement.set('name', self.name)
191 parmElement.set('name', self.name)
243 parmElement.set('value', self.value)
192 parmElement.set('value', self.value)
244 parmElement.set('format', self.format)
193 parmElement.set('format', self.format)
245
194
246 def readXml(self, parmElement):
195 def readXml(self, parmElement):
247
196
248 self.id = parmElement.get('id')
197 self.id = parmElement.get('id')
249 self.name = parmElement.get('name')
198 self.name = parmElement.get('name')
250 self.value = parmElement.get('value')
199 self.value = parmElement.get('value')
251 self.format = str.lower(parmElement.get('format'))
200 self.format = str.lower(parmElement.get('format'))
252
201
253 #Compatible with old signal chain version
202 #Compatible with old signal chain version
254 if self.format == 'int' and self.name == 'idfigure':
203 if self.format == 'int' and self.name == 'idfigure':
255 self.name = 'id'
204 self.name = 'id'
256
205
257 def printattr(self):
206 def printattr(self):
258
207
259 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
208 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
260
209
261 class OperationConf():
210 class OperationConf():
262
211
263 id = None
212 id = None
264 name = None
213 name = None
265 priority = None
214 priority = None
266 type = None
215 type = None
267
216
268 parmConfObjList = []
217 parmConfObjList = []
269
218
270 ELEMENTNAME = 'Operation'
219 ELEMENTNAME = 'Operation'
271
220
272 def __init__(self):
221 def __init__(self):
273
222
274 self.id = '0'
223 self.id = '0'
275 self.name = None
224 self.name = None
276 self.priority = None
225 self.priority = None
277 self.type = 'self'
226 self.type = 'self'
278
227
279
228
280 def __getNewId(self):
229 def __getNewId(self):
281
230
282 return int(self.id)*10 + len(self.parmConfObjList) + 1
231 return int(self.id)*10 + len(self.parmConfObjList) + 1
283
232
284 def updateId(self, new_id):
233 def updateId(self, new_id):
285
234
286 self.id = str(new_id)
235 self.id = str(new_id)
287
236
288 n = 1
237 n = 1
289 for parmObj in self.parmConfObjList:
238 for parmObj in self.parmConfObjList:
290
239
291 idParm = str(int(new_id)*10 + n)
240 idParm = str(int(new_id)*10 + n)
292 parmObj.updateId(idParm)
241 parmObj.updateId(idParm)
293
242
294 n += 1
243 n += 1
295
244
296 def getElementName(self):
245 def getElementName(self):
297
246
298 return self.ELEMENTNAME
247 return self.ELEMENTNAME
299
248
300 def getParameterObjList(self):
249 def getParameterObjList(self):
301
250
302 return self.parmConfObjList
251 return self.parmConfObjList
303
252
304 def getParameterObj(self, parameterName):
253 def getParameterObj(self, parameterName):
305
254
306 for parmConfObj in self.parmConfObjList:
255 for parmConfObj in self.parmConfObjList:
307
256
308 if parmConfObj.name != parameterName:
257 if parmConfObj.name != parameterName:
309 continue
258 continue
310
259
311 return parmConfObj
260 return parmConfObj
312
261
313 return None
262 return None
314
263
315 def getParameterObjfromValue(self, parameterValue):
264 def getParameterObjfromValue(self, parameterValue):
316
265
317 for parmConfObj in self.parmConfObjList:
266 for parmConfObj in self.parmConfObjList:
318
267
319 if parmConfObj.getValue() != parameterValue:
268 if parmConfObj.getValue() != parameterValue:
320 continue
269 continue
321
270
322 return parmConfObj.getValue()
271 return parmConfObj.getValue()
323
272
324 return None
273 return None
325
274
326 def getParameterValue(self, parameterName):
275 def getParameterValue(self, parameterName):
327
276
328 parameterObj = self.getParameterObj(parameterName)
277 parameterObj = self.getParameterObj(parameterName)
329
278
330 # if not parameterObj:
279 # if not parameterObj:
331 # return None
280 # return None
332
281
333 value = parameterObj.getValue()
282 value = parameterObj.getValue()
334
283
335 return value
284 return value
336
285
337
338 def getKwargs(self):
339
340 kwargs = {}
341
342 for parmConfObj in self.parmConfObjList:
343 if self.name == 'run' and parmConfObj.name == 'datatype':
344 continue
345
346 kwargs[parmConfObj.name] = parmConfObj.getValue()
347
348 return kwargs
349
350 def setup(self, id, name, priority, type):
286 def setup(self, id, name, priority, type):
351
287
352 self.id = str(id)
288 self.id = str(id)
353 self.name = name
289 self.name = name
354 self.type = type
290 self.type = type
355 self.priority = priority
291 self.priority = priority
356
292
357 self.parmConfObjList = []
293 self.parmConfObjList = []
358
294
359 def removeParameters(self):
295 def removeParameters(self):
360
296
361 for obj in self.parmConfObjList:
297 for obj in self.parmConfObjList:
362 del obj
298 del obj
363
299
364 self.parmConfObjList = []
300 self.parmConfObjList = []
365
301
366 def addParameter(self, name, value, format='str'):
302 def addParameter(self, name, value, format='str'):
367
303
368 id = self.__getNewId()
304 id = self.__getNewId()
369
305
370 parmConfObj = ParameterConf()
306 parmConfObj = ParameterConf()
371 if not parmConfObj.setup(id, name, value, format):
307 if not parmConfObj.setup(id, name, value, format):
372 return None
308 return None
373
309
374 self.parmConfObjList.append(parmConfObj)
310 self.parmConfObjList.append(parmConfObj)
375
311
376 return parmConfObj
312 return parmConfObj
377
313
378 def changeParameter(self, name, value, format='str'):
314 def changeParameter(self, name, value, format='str'):
379
315
380 parmConfObj = self.getParameterObj(name)
316 parmConfObj = self.getParameterObj(name)
381 parmConfObj.update(name, value, format)
317 parmConfObj.update(name, value, format)
382
318
383 return parmConfObj
319 return parmConfObj
384
320
385 def makeXml(self, procUnitElement):
321 def makeXml(self, procUnitElement):
386
322
387 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
323 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
388 opElement.set('id', str(self.id))
324 opElement.set('id', str(self.id))
389 opElement.set('name', self.name)
325 opElement.set('name', self.name)
390 opElement.set('type', self.type)
326 opElement.set('type', self.type)
391 opElement.set('priority', str(self.priority))
327 opElement.set('priority', str(self.priority))
392
328
393 for parmConfObj in self.parmConfObjList:
329 for parmConfObj in self.parmConfObjList:
394 parmConfObj.makeXml(opElement)
330 parmConfObj.makeXml(opElement)
395
331
396 def readXml(self, opElement):
332 def readXml(self, opElement):
397
333
398 self.id = opElement.get('id')
334 self.id = opElement.get('id')
399 self.name = opElement.get('name')
335 self.name = opElement.get('name')
400 self.type = opElement.get('type')
336 self.type = opElement.get('type')
401 self.priority = opElement.get('priority')
337 self.priority = opElement.get('priority')
402
338
403 #Compatible with old signal chain version
339 #Compatible with old signal chain version
404 #Use of 'run' method instead 'init'
340 #Use of 'run' method instead 'init'
405 if self.type == 'self' and self.name == 'init':
341 if self.type == 'self' and self.name == 'init':
406 self.name = 'run'
342 self.name = 'run'
407
343
408 self.parmConfObjList = []
344 self.parmConfObjList = []
409
345
410 parmElementList = opElement.iter(ParameterConf().getElementName())
346 parmElementList = opElement.iter(ParameterConf().getElementName())
411
347
412 for parmElement in parmElementList:
348 for parmElement in parmElementList:
413 parmConfObj = ParameterConf()
349 parmConfObj = ParameterConf()
414 parmConfObj.readXml(parmElement)
350 parmConfObj.readXml(parmElement)
415
351
416 #Compatible with old signal chain version
352 #Compatible with old signal chain version
417 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
353 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
418 if self.type != 'self' and self.name == 'Plot':
354 if self.type != 'self' and self.name == 'Plot':
419 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
355 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
420 self.name = parmConfObj.value
356 self.name = parmConfObj.value
421 continue
357 continue
422
358
423 self.parmConfObjList.append(parmConfObj)
359 self.parmConfObjList.append(parmConfObj)
424
360
425 def printattr(self):
361 def printattr(self):
426
362
427 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
363 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
428 self.id,
364 self.id,
429 self.name,
365 self.name,
430 self.type,
366 self.type,
431 self.priority)
367 self.priority)
432
368
433 for parmConfObj in self.parmConfObjList:
369 for parmConfObj in self.parmConfObjList:
434 parmConfObj.printattr()
370 parmConfObj.printattr()
435
371
436 def createObject(self, plotter_queue=None):
372 def createObject(self, plotter_queue=None):
437
373
438
439 if self.type == 'self':
374 if self.type == 'self':
440 raise ValueError, "This operation type cannot be created"
375 raise ValueError, "This operation type cannot be created"
441
376
442 if self.type == 'plotter':
377 if self.type == 'plotter':
443 #Plotter(plotter_name)
378 #Plotter(plotter_name)
444 if not plotter_queue:
379 if not plotter_queue:
445 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
380 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
446
381
447 opObj = Plotter(self.name, plotter_queue)
382 opObj = Plotter(self.name, plotter_queue)
448
383
449 if self.type == 'external' or self.type == 'other':
384 if self.type == 'external' or self.type == 'other':
450
451 className = eval(self.name)
385 className = eval(self.name)
452 kwargs = self.getKwargs()
386 opObj = className()
453
387
454 opObj = className(**kwargs)
455
456 return opObj
388 return opObj
457
389
458
459 class ProcUnitConf():
390 class ProcUnitConf():
460
391
461 id = None
392 id = None
462 name = None
393 name = None
463 datatype = None
394 datatype = None
464 inputId = None
395 inputId = None
465 parentId = None
396 parentId = None
466
397
467 opConfObjList = []
398 opConfObjList = []
468
399
469 procUnitObj = None
400 procUnitObj = None
470 opObjList = []
401 opObjList = []
471
402
472 ELEMENTNAME = 'ProcUnit'
403 ELEMENTNAME = 'ProcUnit'
473
404
474 def __init__(self):
405 def __init__(self):
475
406
476 self.id = None
407 self.id = None
477 self.datatype = None
408 self.datatype = None
478 self.name = None
409 self.name = None
479 self.inputId = None
410 self.inputId = None
480
411
481 self.opConfObjList = []
412 self.opConfObjList = []
482
413
483 self.procUnitObj = None
414 self.procUnitObj = None
484 self.opObjDict = {}
415 self.opObjDict = {}
485
416
486 def __getPriority(self):
417 def __getPriority(self):
487
418
488 return len(self.opConfObjList)+1
419 return len(self.opConfObjList)+1
489
420
490 def __getNewId(self):
421 def __getNewId(self):
491
422
492 return int(self.id)*10 + len(self.opConfObjList) + 1
423 return int(self.id)*10 + len(self.opConfObjList) + 1
493
424
494 def getElementName(self):
425 def getElementName(self):
495
426
496 return self.ELEMENTNAME
427 return self.ELEMENTNAME
497
428
498 def getId(self):
429 def getId(self):
499
430
500 return self.id
431 return self.id
501
432
502 def updateId(self, new_id, parentId=parentId):
433 def updateId(self, new_id, parentId=parentId):
503
434
504
435
505 new_id = int(parentId)*10 + (int(self.id) % 10)
436 new_id = int(parentId)*10 + (int(self.id) % 10)
506 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
437 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
507
438
508 #If this proc unit has not inputs
439 #If this proc unit has not inputs
509 if self.inputId == '0':
440 if self.inputId == '0':
510 new_inputId = 0
441 new_inputId = 0
511
442
512 n = 1
443 n = 1
513 for opConfObj in self.opConfObjList:
444 for opConfObj in self.opConfObjList:
514
445
515 idOp = str(int(new_id)*10 + n)
446 idOp = str(int(new_id)*10 + n)
516 opConfObj.updateId(idOp)
447 opConfObj.updateId(idOp)
517
448
518 n += 1
449 n += 1
519
450
520 self.parentId = str(parentId)
451 self.parentId = str(parentId)
521 self.id = str(new_id)
452 self.id = str(new_id)
522 self.inputId = str(new_inputId)
453 self.inputId = str(new_inputId)
523
454
524
455
525 def getInputId(self):
456 def getInputId(self):
526
457
527 return self.inputId
458 return self.inputId
528
459
529 def getOperationObjList(self):
460 def getOperationObjList(self):
530
461
531 return self.opConfObjList
462 return self.opConfObjList
532
463
533 def getOperationObj(self, name=None):
464 def getOperationObj(self, name=None):
534
465
535 for opConfObj in self.opConfObjList:
466 for opConfObj in self.opConfObjList:
536
467
537 if opConfObj.name != name:
468 if opConfObj.name != name:
538 continue
469 continue
539
470
540 return opConfObj
471 return opConfObj
541
472
542 return None
473 return None
543
474
544 def getOpObjfromParamValue(self, value=None):
475 def getOpObjfromParamValue(self, value=None):
545
476
546 for opConfObj in self.opConfObjList:
477 for opConfObj in self.opConfObjList:
547 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
478 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
548 continue
479 continue
549 return opConfObj
480 return opConfObj
550 return None
481 return None
551
482
552 def getProcUnitObj(self):
483 def getProcUnitObj(self):
553
484
554 return self.procUnitObj
485 return self.procUnitObj
555
486
556 def setup(self, id, name, datatype, inputId, parentId=None):
487 def setup(self, id, name, datatype, inputId, parentId=None):
557
488
558 #Compatible with old signal chain version
489 #Compatible with old signal chain version
559 if datatype==None and name==None:
490 if datatype==None and name==None:
560 raise ValueError, "datatype or name should be defined"
491 raise ValueError, "datatype or name should be defined"
561
492
562 if name==None:
493 if name==None:
563 if 'Proc' in datatype:
494 if 'Proc' in datatype:
564 name = datatype
495 name = datatype
565 else:
496 else:
566 name = '%sProc' %(datatype)
497 name = '%sProc' %(datatype)
567
498
568 if datatype==None:
499 if datatype==None:
569 datatype = name.replace('Proc','')
500 datatype = name.replace('Proc','')
570
501
571 self.id = str(id)
502 self.id = str(id)
572 self.name = name
503 self.name = name
573 self.datatype = datatype
504 self.datatype = datatype
574 self.inputId = inputId
505 self.inputId = inputId
575 self.parentId = parentId
506 self.parentId = parentId
576
507
577 self.opConfObjList = []
508 self.opConfObjList = []
578
509
579 self.addOperation(name='run', optype='self')
510 self.addOperation(name='run', optype='self')
580
511
581 def removeOperations(self):
512 def removeOperations(self):
582
513
583 for obj in self.opConfObjList:
514 for obj in self.opConfObjList:
584 del obj
515 del obj
585
516
586 self.opConfObjList = []
517 self.opConfObjList = []
587 self.addOperation(name='run')
518 self.addOperation(name='run')
588
519
589 def addParameter(self, **kwargs):
520 def addParameter(self, **kwargs):
590 '''
521 '''
591 Add parameters to "run" operation
522 Add parameters to "run" operation
592 '''
523 '''
593 opObj = self.opConfObjList[0]
524 opObj = self.opConfObjList[0]
594
525
595 opObj.addParameter(**kwargs)
526 opObj.addParameter(**kwargs)
596
527
597 return opObj
528 return opObj
598
529
599 def addOperation(self, name, optype='self'):
530 def addOperation(self, name, optype='self'):
600
531
601 id = self.__getNewId()
532 id = self.__getNewId()
602 priority = self.__getPriority()
533 priority = self.__getPriority()
603
534
604 opConfObj = OperationConf()
535 opConfObj = OperationConf()
605 opConfObj.setup(id, name=name, priority=priority, type=optype)
536 opConfObj.setup(id, name=name, priority=priority, type=optype)
606
537
607 self.opConfObjList.append(opConfObj)
538 self.opConfObjList.append(opConfObj)
608
539
609 return opConfObj
540 return opConfObj
610
541
611 def makeXml(self, projectElement):
542 def makeXml(self, projectElement):
612
543
613 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
544 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
614 procUnitElement.set('id', str(self.id))
545 procUnitElement.set('id', str(self.id))
615 procUnitElement.set('name', self.name)
546 procUnitElement.set('name', self.name)
616 procUnitElement.set('datatype', self.datatype)
547 procUnitElement.set('datatype', self.datatype)
617 procUnitElement.set('inputId', str(self.inputId))
548 procUnitElement.set('inputId', str(self.inputId))
618
549
619 for opConfObj in self.opConfObjList:
550 for opConfObj in self.opConfObjList:
620 opConfObj.makeXml(procUnitElement)
551 opConfObj.makeXml(procUnitElement)
621
552
622 def readXml(self, upElement):
553 def readXml(self, upElement):
623
554
624 self.id = upElement.get('id')
555 self.id = upElement.get('id')
625 self.name = upElement.get('name')
556 self.name = upElement.get('name')
626 self.datatype = upElement.get('datatype')
557 self.datatype = upElement.get('datatype')
627 self.inputId = upElement.get('inputId')
558 self.inputId = upElement.get('inputId')
628
559
629 if self.ELEMENTNAME == "ReadUnit":
560 if self.ELEMENTNAME == "ReadUnit":
630 self.datatype = self.datatype.replace("Reader", "")
561 self.datatype = self.datatype.replace("Reader", "")
631
562
632 if self.ELEMENTNAME == "ProcUnit":
563 if self.ELEMENTNAME == "ProcUnit":
633 self.datatype = self.datatype.replace("Proc", "")
564 self.datatype = self.datatype.replace("Proc", "")
634
565
635 if self.inputId == 'None':
566 if self.inputId == 'None':
636 self.inputId = '0'
567 self.inputId = '0'
637
568
638 self.opConfObjList = []
569 self.opConfObjList = []
639
570
640 opElementList = upElement.iter(OperationConf().getElementName())
571 opElementList = upElement.iter(OperationConf().getElementName())
641
572
642 for opElement in opElementList:
573 for opElement in opElementList:
643 opConfObj = OperationConf()
574 opConfObj = OperationConf()
644 opConfObj.readXml(opElement)
575 opConfObj.readXml(opElement)
645 self.opConfObjList.append(opConfObj)
576 self.opConfObjList.append(opConfObj)
646
577
647 def printattr(self):
578 def printattr(self):
648
579
649 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
580 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
650 self.id,
581 self.id,
651 self.name,
582 self.name,
@@ -654,54 +585,41 class ProcUnitConf():
654
585
655 for opConfObj in self.opConfObjList:
586 for opConfObj in self.opConfObjList:
656 opConfObj.printattr()
587 opConfObj.printattr()
657
588
658
659 def getKwargs(self):
660
661 opObj = self.opConfObjList[0]
662 kwargs = opObj.getKwargs()
663
664 return kwargs
665
666 def createObjects(self, plotter_queue=None):
589 def createObjects(self, plotter_queue=None):
667
590
668 className = eval(self.name)
591 className = eval(self.name)
669 kwargs = self.getKwargs()
592 procUnitObj = className()
670 procUnitObj = className(**kwargs)
593
671
672 for opConfObj in self.opConfObjList:
594 for opConfObj in self.opConfObjList:
673
595
674 if opConfObj.type=='self' and self.name=='run':
596 if opConfObj.type == 'self':
675 continue
676 elif opConfObj.type=='self':
677 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
678 continue
597 continue
679
598
680 opObj = opConfObj.createObject(plotter_queue)
599 opObj = opConfObj.createObject(plotter_queue)
681
600
682 self.opObjDict[opConfObj.id] = opObj
601 self.opObjDict[opConfObj.id] = opObj
683
684 procUnitObj.addOperation(opObj, opConfObj.id)
602 procUnitObj.addOperation(opObj, opConfObj.id)
685
603
686 self.procUnitObj = procUnitObj
604 self.procUnitObj = procUnitObj
687
605
688 return procUnitObj
606 return procUnitObj
689
607
690 def run(self):
608 def run(self):
691
609
692 is_ok = False
610 is_ok = False
693
611
694 for opConfObj in self.opConfObjList:
612 for opConfObj in self.opConfObjList:
695
613
696 kwargs = {}
614 kwargs = {}
697 for parmConfObj in opConfObj.getParameterObjList():
615 for parmConfObj in opConfObj.getParameterObjList():
698 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
616 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
699 continue
617 continue
700
618
701 kwargs[parmConfObj.name] = parmConfObj.getValue()
619 kwargs[parmConfObj.name] = parmConfObj.getValue()
702
620
703 #ini = time.time()
621 #ini = time.time()
704
622
705 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
623 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
706 sts = self.procUnitObj.call(opType = opConfObj.type,
624 sts = self.procUnitObj.call(opType = opConfObj.type,
707 opName = opConfObj.name,
625 opName = opConfObj.name,
@@ -714,50 +632,49 class ProcUnitConf():
714 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
632 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
715
633
716 is_ok = is_ok or sts
634 is_ok = is_ok or sts
717
635
718 return is_ok
636 return is_ok
719
637
720 def close(self):
638 def close(self):
721
639
722 for opConfObj in self.opConfObjList:
640 for opConfObj in self.opConfObjList:
723 if opConfObj.type == 'self':
641 if opConfObj.type == 'self':
724 continue
642 continue
725
643
726 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
644 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
727 opObj.close()
645 opObj.close()
728
646
729 self.procUnitObj.close()
647 self.procUnitObj.close()
730
648
731 return
649 return
732
650
733 class ReadUnitConf(ProcUnitConf):
651 class ReadUnitConf(ProcUnitConf):
734
652
735 path = None
653 path = None
736 startDate = None
654 startDate = None
737 endDate = None
655 endDate = None
738 startTime = None
656 startTime = None
739 endTime = None
657 endTime = None
740
658
741 ELEMENTNAME = 'ReadUnit'
659 ELEMENTNAME = 'ReadUnit'
742
660
743 def __init__(self):
661 def __init__(self):
744
662
745 self.id = None
663 self.id = None
746 self.datatype = None
664 self.datatype = None
747 self.name = None
665 self.name = None
748 self.inputId = None
666 self.inputId = None
749
667
750 self.parentId = None
668 self.parentId = None
751
669
752 self.opConfObjList = []
670 self.opConfObjList = []
753 self.opObjList = []
671 self.opObjList = []
754
672
755 def getElementName(self):
673 def getElementName(self):
756
674
757 return self.ELEMENTNAME
675 return self.ELEMENTNAME
758
676
759 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
677 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, **kwargs):
760 endTime="", parentId=None, queue=None, server=None, **kwargs):
761
678
762 #Compatible with old signal chain version
679 #Compatible with old signal chain version
763 if datatype==None and name==None:
680 if datatype==None and name==None:
@@ -768,40 +685,40 class ReadUnitConf(ProcUnitConf):
768 name = datatype
685 name = datatype
769 else:
686 else:
770 name = '%sReader' %(datatype)
687 name = '%sReader' %(datatype)
688
771 if datatype==None:
689 if datatype==None:
772 datatype = name.replace('Reader','')
690 datatype = name.replace('Reader','')
773
691
774 self.id = id
692 self.id = id
775 self.name = name
693 self.name = name
776 self.datatype = datatype
694 self.datatype = datatype
777 if path != '':
695
778 self.path = os.path.abspath(path)
696 self.path = os.path.abspath(path)
779 self.startDate = startDate
697 self.startDate = startDate
780 self.endDate = endDate
698 self.endDate = endDate
781 self.startTime = startTime
699 self.startTime = startTime
782 self.endTime = endTime
700 self.endTime = endTime
783
701
784 self.inputId = '0'
702 self.inputId = '0'
785 self.parentId = parentId
703 self.parentId = parentId
786 self.queue = queue
704
787 self.server = server
788 self.addRunOperation(**kwargs)
705 self.addRunOperation(**kwargs)
789
706
790 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
707 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
791
708
792 #Compatible with old signal chain version
709 #Compatible with old signal chain version
793 if datatype==None and name==None:
710 if datatype==None and name==None:
794 raise ValueError, "datatype or name should be defined"
711 raise ValueError, "datatype or name should be defined"
795
712
796 if name==None:
713 if name==None:
797 if 'Reader' in datatype:
714 if 'Reader' in datatype:
798 name = datatype
715 name = datatype
799 else:
716 else:
800 name = '%sReader' %(datatype)
717 name = '%sReader' %(datatype)
801
718
802 if datatype==None:
719 if datatype==None:
803 datatype = name.replace('Reader','')
720 datatype = name.replace('Reader','')
804
721
805 self.datatype = datatype
722 self.datatype = datatype
806 self.name = name
723 self.name = name
807 self.path = path
724 self.path = path
@@ -809,54 +726,50 class ReadUnitConf(ProcUnitConf):
809 self.endDate = endDate
726 self.endDate = endDate
810 self.startTime = startTime
727 self.startTime = startTime
811 self.endTime = endTime
728 self.endTime = endTime
812
729
813 self.inputId = '0'
730 self.inputId = '0'
814 self.parentId = parentId
731 self.parentId = parentId
815
732
816 self.updateRunOperation(**kwargs)
733 self.updateRunOperation(**kwargs)
817
734
818 def removeOperations(self):
735 def removeOperations(self):
819
736
820 for obj in self.opConfObjList:
737 for obj in self.opConfObjList:
821 del obj
738 del obj
822
739
823 self.opConfObjList = []
740 self.opConfObjList = []
824
741
825 def addRunOperation(self, **kwargs):
742 def addRunOperation(self, **kwargs):
826
743
827 opObj = self.addOperation(name = 'run', optype = 'self')
744 opObj = self.addOperation(name = 'run', optype = 'self')
828
829 if self.server is None:
830 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
831 opObj.addParameter(name='path' , value=self.path, format='str')
832 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
833 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
834 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
835 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
836 opObj.addParameter(name='queue' , value=self.queue, format='obj')
837 for key, value in kwargs.items():
838 opObj.addParameter(name=key, value=value, format=type(value).__name__)
839 else:
840 opObj.addParameter(name='server' , value=self.server, format='str')
841
745
842
746 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
747 opObj.addParameter(name='path' , value=self.path, format='str')
748 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
749 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
750 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
751 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
752
753 for key, value in kwargs.items():
754 opObj.addParameter(name=key, value=value, format=type(value).__name__)
755
843 return opObj
756 return opObj
844
757
845 def updateRunOperation(self, **kwargs):
758 def updateRunOperation(self, **kwargs):
846
759
847 opObj = self.getOperationObj(name = 'run')
760 opObj = self.getOperationObj(name = 'run')
848 opObj.removeParameters()
761 opObj.removeParameters()
849
762
850 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
763 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
851 opObj.addParameter(name='path' , value=self.path, format='str')
764 opObj.addParameter(name='path' , value=self.path, format='str')
852 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
765 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
853 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
766 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
854 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
767 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
855 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
768 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
856
769
857 for key, value in kwargs.items():
770 for key, value in kwargs.items():
858 opObj.addParameter(name=key, value=value, format=type(value).__name__)
771 opObj.addParameter(name=key, value=value, format=type(value).__name__)
859
772
860 return opObj
773 return opObj
861
774
862 # def makeXml(self, projectElement):
775 # def makeXml(self, projectElement):
@@ -871,314 +784,314 class ReadUnitConf(ProcUnitConf):
871 # opConfObj.makeXml(procUnitElement)
784 # opConfObj.makeXml(procUnitElement)
872
785
873 def readXml(self, upElement):
786 def readXml(self, upElement):
874
787
875 self.id = upElement.get('id')
788 self.id = upElement.get('id')
876 self.name = upElement.get('name')
789 self.name = upElement.get('name')
877 self.datatype = upElement.get('datatype')
790 self.datatype = upElement.get('datatype')
878 self.inputId = upElement.get('inputId')
791 self.inputId = upElement.get('inputId')
879
792
880 if self.ELEMENTNAME == "ReadUnit":
793 if self.ELEMENTNAME == "ReadUnit":
881 self.datatype = self.datatype.replace("Reader", "")
794 self.datatype = self.datatype.replace("Reader", "")
882
795
883 if self.inputId == 'None':
796 if self.inputId == 'None':
884 self.inputId = '0'
797 self.inputId = '0'
885
798
886 self.opConfObjList = []
799 self.opConfObjList = []
887
800
888 opElementList = upElement.iter(OperationConf().getElementName())
801 opElementList = upElement.iter(OperationConf().getElementName())
889
802
890 for opElement in opElementList:
803 for opElement in opElementList:
891 opConfObj = OperationConf()
804 opConfObj = OperationConf()
892 opConfObj.readXml(opElement)
805 opConfObj.readXml(opElement)
893 self.opConfObjList.append(opConfObj)
806 self.opConfObjList.append(opConfObj)
894
807
895 if opConfObj.name == 'run':
808 if opConfObj.name == 'run':
896 self.path = opConfObj.getParameterValue('path')
809 self.path = opConfObj.getParameterValue('path')
897 self.startDate = opConfObj.getParameterValue('startDate')
810 self.startDate = opConfObj.getParameterValue('startDate')
898 self.endDate = opConfObj.getParameterValue('endDate')
811 self.endDate = opConfObj.getParameterValue('endDate')
899 self.startTime = opConfObj.getParameterValue('startTime')
812 self.startTime = opConfObj.getParameterValue('startTime')
900 self.endTime = opConfObj.getParameterValue('endTime')
813 self.endTime = opConfObj.getParameterValue('endTime')
901
814
902 class Project():
815 class Project():
903
816
904 id = None
817 id = None
905 name = None
818 name = None
906 description = None
819 description = None
907 filename = None
820 filename = None
908
821
909 procUnitConfObjDict = None
822 procUnitConfObjDict = None
910
823
911 ELEMENTNAME = 'Project'
824 ELEMENTNAME = 'Project'
912
825
913 plotterQueue = None
826 plotterQueue = None
914
827
915 def __init__(self, plotter_queue=None):
828 def __init__(self, plotter_queue=None):
916
829
917 self.id = None
830 self.id = None
918 self.name = None
831 self.name = None
919 self.description = None
832 self.description = None
920
833
921 self.plotterQueue = plotter_queue
834 self.plotterQueue = plotter_queue
922
835
923 self.procUnitConfObjDict = {}
836 self.procUnitConfObjDict = {}
924
837
925 def __getNewId(self):
838 def __getNewId(self):
926
839
927 idList = self.procUnitConfObjDict.keys()
840 idList = self.procUnitConfObjDict.keys()
928
841
929 id = int(self.id)*10
842 id = int(self.id)*10
930
843
931 while True:
844 while True:
932 id += 1
845 id += 1
933
846
934 if str(id) in idList:
847 if str(id) in idList:
935 continue
848 continue
936
849
937 break
850 break
938
851
939 return str(id)
852 return str(id)
940
853
941 def getElementName(self):
854 def getElementName(self):
942
855
943 return self.ELEMENTNAME
856 return self.ELEMENTNAME
944
857
945 def getId(self):
858 def getId(self):
946
859
947 return self.id
860 return self.id
948
861
949 def updateId(self, new_id):
862 def updateId(self, new_id):
950
863
951 self.id = str(new_id)
864 self.id = str(new_id)
952
865
953 keyList = self.procUnitConfObjDict.keys()
866 keyList = self.procUnitConfObjDict.keys()
954 keyList.sort()
867 keyList.sort()
955
868
956 n = 1
869 n = 1
957 newProcUnitConfObjDict = {}
870 newProcUnitConfObjDict = {}
958
871
959 for procKey in keyList:
872 for procKey in keyList:
960
873
961 procUnitConfObj = self.procUnitConfObjDict[procKey]
874 procUnitConfObj = self.procUnitConfObjDict[procKey]
962 idProcUnit = str(int(self.id)*10 + n)
875 idProcUnit = str(int(self.id)*10 + n)
963 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
876 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
964
877
965 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
878 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
966 n += 1
879 n += 1
967
880
968 self.procUnitConfObjDict = newProcUnitConfObjDict
881 self.procUnitConfObjDict = newProcUnitConfObjDict
969
882
970 def setup(self, id, name, description):
883 def setup(self, id, name, description):
971
884
972 self.id = str(id)
885 self.id = str(id)
973 self.name = name
886 self.name = name
974 self.description = description
887 self.description = description
975
888
976 def update(self, name, description):
889 def update(self, name, description):
977
890
978 self.name = name
891 self.name = name
979 self.description = description
892 self.description = description
980
893
981 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
894 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
982
895
983 if id is None:
896 if id is None:
984 idReadUnit = self.__getNewId()
897 idReadUnit = self.__getNewId()
985 else:
898 else:
986 idReadUnit = str(id)
899 idReadUnit = str(id)
987
900
988 readUnitConfObj = ReadUnitConf()
901 readUnitConfObj = ReadUnitConf()
989 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
902 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
990
903
991 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
904 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
992
905
993 return readUnitConfObj
906 return readUnitConfObj
994
907
995 def addProcUnit(self, inputId='0', datatype=None, name=None):
908 def addProcUnit(self, inputId='0', datatype=None, name=None):
996
909
997 idProcUnit = self.__getNewId()
910 idProcUnit = self.__getNewId()
998
911
999 procUnitConfObj = ProcUnitConf()
912 procUnitConfObj = ProcUnitConf()
1000 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
913 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1001
914
1002 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
915 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1003
916
1004 return procUnitConfObj
917 return procUnitConfObj
1005
918
1006 def removeProcUnit(self, id):
919 def removeProcUnit(self, id):
1007
920
1008 if id in self.procUnitConfObjDict.keys():
921 if id in self.procUnitConfObjDict.keys():
1009 self.procUnitConfObjDict.pop(id)
922 self.procUnitConfObjDict.pop(id)
1010
923
1011 def getReadUnitId(self):
924 def getReadUnitId(self):
1012
925
1013 readUnitConfObj = self.getReadUnitObj()
926 readUnitConfObj = self.getReadUnitObj()
1014
927
1015 return readUnitConfObj.id
928 return readUnitConfObj.id
1016
929
1017 def getReadUnitObj(self):
930 def getReadUnitObj(self):
1018
931
1019 for obj in self.procUnitConfObjDict.values():
932 for obj in self.procUnitConfObjDict.values():
1020 if obj.getElementName() == "ReadUnit":
933 if obj.getElementName() == "ReadUnit":
1021 return obj
934 return obj
1022
935
1023 return None
936 return None
1024
937
1025 def getProcUnitObj(self, id=None, name=None):
938 def getProcUnitObj(self, id=None, name=None):
1026
939
1027 if id != None:
940 if id != None:
1028 return self.procUnitConfObjDict[id]
941 return self.procUnitConfObjDict[id]
1029
942
1030 if name != None:
943 if name != None:
1031 return self.getProcUnitObjByName(name)
944 return self.getProcUnitObjByName(name)
1032
945
1033 return None
946 return None
1034
947
1035 def getProcUnitObjByName(self, name):
948 def getProcUnitObjByName(self, name):
1036
949
1037 for obj in self.procUnitConfObjDict.values():
950 for obj in self.procUnitConfObjDict.values():
1038 if obj.name == name:
951 if obj.name == name:
1039 return obj
952 return obj
1040
953
1041 return None
954 return None
1042
955
1043 def procUnitItems(self):
956 def procUnitItems(self):
1044
957
1045 return self.procUnitConfObjDict.items()
958 return self.procUnitConfObjDict.items()
1046
959
1047 def makeXml(self):
960 def makeXml(self):
1048
961
1049 projectElement = Element('Project')
962 projectElement = Element('Project')
1050 projectElement.set('id', str(self.id))
963 projectElement.set('id', str(self.id))
1051 projectElement.set('name', self.name)
964 projectElement.set('name', self.name)
1052 projectElement.set('description', self.description)
965 projectElement.set('description', self.description)
1053
966
1054 for procUnitConfObj in self.procUnitConfObjDict.values():
967 for procUnitConfObj in self.procUnitConfObjDict.values():
1055 procUnitConfObj.makeXml(projectElement)
968 procUnitConfObj.makeXml(projectElement)
1056
969
1057 self.projectElement = projectElement
970 self.projectElement = projectElement
1058
971
1059 def writeXml(self, filename=None):
972 def writeXml(self, filename=None):
1060
973
1061 if filename == None:
974 if filename == None:
1062 if self.filename:
975 if self.filename:
1063 filename = self.filename
976 filename = self.filename
1064 else:
977 else:
1065 filename = "schain.xml"
978 filename = "schain.xml"
1066
979
1067 if not filename:
980 if not filename:
1068 print "filename has not been defined. Use setFilename(filename) for do it."
981 print "filename has not been defined. Use setFilename(filename) for do it."
1069 return 0
982 return 0
1070
983
1071 abs_file = os.path.abspath(filename)
984 abs_file = os.path.abspath(filename)
1072
985
1073 if not os.access(os.path.dirname(abs_file), os.W_OK):
986 if not os.access(os.path.dirname(abs_file), os.W_OK):
1074 print "No write permission on %s" %os.path.dirname(abs_file)
987 print "No write permission on %s" %os.path.dirname(abs_file)
1075 return 0
988 return 0
1076
989
1077 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
990 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1078 print "File %s already exists and it could not be overwriten" %abs_file
991 print "File %s already exists and it could not be overwriten" %abs_file
1079 return 0
992 return 0
1080
993
1081 self.makeXml()
994 self.makeXml()
1082
995
1083 ElementTree(self.projectElement).write(abs_file, method='xml')
996 ElementTree(self.projectElement).write(abs_file, method='xml')
1084
997
1085 self.filename = abs_file
998 self.filename = abs_file
1086
999
1087 return 1
1000 return 1
1088
1001
1089 def readXml(self, filename = None):
1002 def readXml(self, filename = None):
1090
1003
1091 if not filename:
1004 if not filename:
1092 print "filename is not defined"
1005 print "filename is not defined"
1093 return 0
1006 return 0
1094
1007
1095 abs_file = os.path.abspath(filename)
1008 abs_file = os.path.abspath(filename)
1096
1009
1097 if not os.path.isfile(abs_file):
1010 if not os.path.isfile(abs_file):
1098 print "%s file does not exist" %abs_file
1011 print "%s file does not exist" %abs_file
1099 return 0
1012 return 0
1100
1013
1101 self.projectElement = None
1014 self.projectElement = None
1102 self.procUnitConfObjDict = {}
1015 self.procUnitConfObjDict = {}
1103
1016
1104 try:
1017 try:
1105 self.projectElement = ElementTree().parse(abs_file)
1018 self.projectElement = ElementTree().parse(abs_file)
1106 except:
1019 except:
1107 print "Error reading %s, verify file format" %filename
1020 print "Error reading %s, verify file format" %filename
1108 return 0
1021 return 0
1109
1022
1110 self.project = self.projectElement.tag
1023 self.project = self.projectElement.tag
1111
1024
1112 self.id = self.projectElement.get('id')
1025 self.id = self.projectElement.get('id')
1113 self.name = self.projectElement.get('name')
1026 self.name = self.projectElement.get('name')
1114 self.description = self.projectElement.get('description')
1027 self.description = self.projectElement.get('description')
1115
1028
1116 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1029 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1117
1030
1118 for readUnitElement in readUnitElementList:
1031 for readUnitElement in readUnitElementList:
1119 readUnitConfObj = ReadUnitConf()
1032 readUnitConfObj = ReadUnitConf()
1120 readUnitConfObj.readXml(readUnitElement)
1033 readUnitConfObj.readXml(readUnitElement)
1121
1034
1122 if readUnitConfObj.parentId == None:
1035 if readUnitConfObj.parentId == None:
1123 readUnitConfObj.parentId = self.id
1036 readUnitConfObj.parentId = self.id
1124
1037
1125 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1038 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1126
1039
1127 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1040 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1128
1041
1129 for procUnitElement in procUnitElementList:
1042 for procUnitElement in procUnitElementList:
1130 procUnitConfObj = ProcUnitConf()
1043 procUnitConfObj = ProcUnitConf()
1131 procUnitConfObj.readXml(procUnitElement)
1044 procUnitConfObj.readXml(procUnitElement)
1132
1045
1133 if procUnitConfObj.parentId == None:
1046 if procUnitConfObj.parentId == None:
1134 procUnitConfObj.parentId = self.id
1047 procUnitConfObj.parentId = self.id
1135
1048
1136 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1049 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1137
1050
1138 self.filename = abs_file
1051 self.filename = abs_file
1139
1052
1140 return 1
1053 return 1
1141
1054
1142 def printattr(self):
1055 def printattr(self):
1143
1056
1144 print "Project[%s]: name = %s, description = %s" %(self.id,
1057 print "Project[%s]: name = %s, description = %s" %(self.id,
1145 self.name,
1058 self.name,
1146 self.description)
1059 self.description)
1147
1060
1148 for procUnitConfObj in self.procUnitConfObjDict.values():
1061 for procUnitConfObj in self.procUnitConfObjDict.values():
1149 procUnitConfObj.printattr()
1062 procUnitConfObj.printattr()
1150
1063
1151 def createObjects(self):
1064 def createObjects(self):
1152
1065
1153 for procUnitConfObj in self.procUnitConfObjDict.values():
1066 for procUnitConfObj in self.procUnitConfObjDict.values():
1154 procUnitConfObj.createObjects(self.plotterQueue)
1067 procUnitConfObj.createObjects(self.plotterQueue)
1155
1068
1156 def __connect(self, objIN, thisObj):
1069 def __connect(self, objIN, thisObj):
1157
1070
1158 thisObj.setInput(objIN.getOutputObj())
1071 thisObj.setInput(objIN.getOutputObj())
1159
1072
1160 def connectObjects(self):
1073 def connectObjects(self):
1161
1074
1162 for thisPUConfObj in self.procUnitConfObjDict.values():
1075 for thisPUConfObj in self.procUnitConfObjDict.values():
1163
1076
1164 inputId = thisPUConfObj.getInputId()
1077 inputId = thisPUConfObj.getInputId()
1165
1078
1166 if int(inputId) == 0:
1079 if int(inputId) == 0:
1167 continue
1080 continue
1168
1081
1169 #Get input object
1082 #Get input object
1170 puConfINObj = self.procUnitConfObjDict[inputId]
1083 puConfINObj = self.procUnitConfObjDict[inputId]
1171 puObjIN = puConfINObj.getProcUnitObj()
1084 puObjIN = puConfINObj.getProcUnitObj()
1172
1085
1173 #Get current object
1086 #Get current object
1174 thisPUObj = thisPUConfObj.getProcUnitObj()
1087 thisPUObj = thisPUConfObj.getProcUnitObj()
1175
1088
1176 self.__connect(puObjIN, thisPUObj)
1089 self.__connect(puObjIN, thisPUObj)
1177
1090
1178 def __handleError(self, procUnitConfObj, send_email=True):
1091 def __handleError(self, procUnitConfObj, send_email=True):
1179
1092
1180 import socket
1093 import socket
1181
1094
1182 err = traceback.format_exception(sys.exc_info()[0],
1095 err = traceback.format_exception(sys.exc_info()[0],
1183 sys.exc_info()[1],
1096 sys.exc_info()[1],
1184 sys.exc_info()[2])
1097 sys.exc_info()[2])
@@ -1187,20 +1100,20 class Project():
1187 print "***** %s" %err[-1]
1100 print "***** %s" %err[-1]
1188
1101
1189 message = "".join(err)
1102 message = "".join(err)
1190
1103
1191 sys.stderr.write(message)
1104 sys.stderr.write(message)
1192
1105
1193 if not send_email:
1106 if not send_email:
1194 return
1107 return
1195
1108
1196 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1109 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1197
1110
1198 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1111 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1199 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1112 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1200 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1113 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1201 subtitle += "Configuration file: %s\n" %self.filename
1114 subtitle += "Configuration file: %s\n" %self.filename
1202 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1115 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1203
1116
1204 readUnitConfObj = self.getReadUnitObj()
1117 readUnitConfObj = self.getReadUnitObj()
1205 if readUnitConfObj:
1118 if readUnitConfObj:
1206 subtitle += "\nInput parameters:\n"
1119 subtitle += "\nInput parameters:\n"
@@ -1210,80 +1123,80 class Project():
1210 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1123 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1211 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1124 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1212 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1125 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1213
1126
1214 adminObj = schainpy.admin.SchainNotify()
1127 adminObj = schainpy.admin.SchainNotify()
1215 adminObj.sendAlert(message=message,
1128 adminObj.sendAlert(message=message,
1216 subject=subject,
1129 subject=subject,
1217 subtitle=subtitle,
1130 subtitle=subtitle,
1218 filename=self.filename)
1131 filename=self.filename)
1219
1132
1220 def isPaused(self):
1133 def isPaused(self):
1221 return 0
1134 return 0
1222
1135
1223 def isStopped(self):
1136 def isStopped(self):
1224 return 0
1137 return 0
1225
1138
1226 def runController(self):
1139 def runController(self):
1227 """
1140 """
1228 returns 0 when this process has been stopped, 1 otherwise
1141 returns 0 when this process has been stopped, 1 otherwise
1229 """
1142 """
1230
1143
1231 if self.isPaused():
1144 if self.isPaused():
1232 print "Process suspended"
1145 print "Process suspended"
1233
1146
1234 while True:
1147 while True:
1235 sleep(0.1)
1148 sleep(0.1)
1236
1149
1237 if not self.isPaused():
1150 if not self.isPaused():
1238 break
1151 break
1239
1152
1240 if self.isStopped():
1153 if self.isStopped():
1241 break
1154 break
1242
1155
1243 print "Process reinitialized"
1156 print "Process reinitialized"
1244
1157
1245 if self.isStopped():
1158 if self.isStopped():
1246 print "Process stopped"
1159 print "Process stopped"
1247 return 0
1160 return 0
1248
1161
1249 return 1
1162 return 1
1250
1163
1251 def setFilename(self, filename):
1164 def setFilename(self, filename):
1252
1165
1253 self.filename = filename
1166 self.filename = filename
1254
1167
1255 def setPlotterQueue(self, plotter_queue):
1168 def setPlotterQueue(self, plotter_queue):
1256
1169
1257 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1170 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1258
1171
1259 def getPlotterQueue(self):
1172 def getPlotterQueue(self):
1260
1173
1261 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1174 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1262
1175
1263 def useExternalPlotter(self):
1176 def useExternalPlotter(self):
1264
1177
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1178 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1266
1179
1267 def run(self):
1180 def run(self):
1268
1181
1269 print
1182 print
1270 print "*"*60
1183 print "*"*60
1271 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1184 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1272 print "*"*60
1185 print "*"*60
1273 print
1186 print
1274
1187
1275 keyList = self.procUnitConfObjDict.keys()
1188 keyList = self.procUnitConfObjDict.keys()
1276 keyList.sort()
1189 keyList.sort()
1277
1190
1278 while(True):
1191 while(True):
1279
1192
1280 is_ok = False
1193 is_ok = False
1281
1194
1282 for procKey in keyList:
1195 for procKey in keyList:
1283 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1196 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1284
1197
1285 procUnitConfObj = self.procUnitConfObjDict[procKey]
1198 procUnitConfObj = self.procUnitConfObjDict[procKey]
1286
1199
1287 try:
1200 try:
1288 sts = procUnitConfObj.run()
1201 sts = procUnitConfObj.run()
1289 is_ok = is_ok or sts
1202 is_ok = is_ok or sts
@@ -1300,7 +1213,7 class Project():
1300 self.__handleError(procUnitConfObj)
1213 self.__handleError(procUnitConfObj)
1301 is_ok = False
1214 is_ok = False
1302 break
1215 break
1303
1216
1304 #If every process unit finished so end process
1217 #If every process unit finished so end process
1305 if not(is_ok):
1218 if not(is_ok):
1306 # print "Every process unit have finished"
1219 # print "Every process unit have finished"
@@ -1308,17 +1221,74 class Project():
1308
1221
1309 if not self.runController():
1222 if not self.runController():
1310 break
1223 break
1311
1224
1312 #Closing every process
1225 #Closing every process
1313 for procKey in keyList:
1226 for procKey in keyList:
1314 procUnitConfObj = self.procUnitConfObjDict[procKey]
1227 procUnitConfObj = self.procUnitConfObjDict[procKey]
1315 procUnitConfObj.close()
1228 procUnitConfObj.close()
1316
1229
1317 print "Process finished"
1230 print "Process finished"
1318
1231
1319 def start(self, filename=None):
1232 def start(self):
1320
1233
1321 self.writeXml(filename)
1234 self.writeXml()
1235
1322 self.createObjects()
1236 self.createObjects()
1323 self.connectObjects()
1237 self.connectObjects()
1324 self.run()
1238 self.run()
1239
1240 if __name__ == '__main__':
1241
1242 desc = "Segundo Test"
1243 filename = "schain.xml"
1244
1245 controllerObj = Project()
1246
1247 controllerObj.setup(id = '191', name='test01', description=desc)
1248
1249 readUnitConfObj = controllerObj.addReadUnit(datatype='Voltage',
1250 path='data/rawdata/',
1251 startDate='2011/01/01',
1252 endDate='2012/12/31',
1253 startTime='00:00:00',
1254 endTime='23:59:59',
1255 online=1,
1256 walk=1)
1257
1258 procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage', inputId=readUnitConfObj.getId())
1259
1260 opObj10 = procUnitConfObj0.addOperation(name='selectChannels')
1261 opObj10.addParameter(name='channelList', value='3,4,5', format='intlist')
1262
1263 opObj10 = procUnitConfObj0.addOperation(name='selectHeights')
1264 opObj10.addParameter(name='minHei', value='90', format='float')
1265 opObj10.addParameter(name='maxHei', value='180', format='float')
1266
1267 opObj12 = procUnitConfObj0.addOperation(name='CohInt', optype='external')
1268 opObj12.addParameter(name='n', value='10', format='int')
1269
1270 procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj0.getId())
1271 procUnitConfObj1.addParameter(name='nFFTPoints', value='32', format='int')
1272 # procUnitConfObj1.addParameter(name='pairList', value='(0,1),(0,2),(1,2)', format='')
1273
1274
1275 opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
1276 opObj11.addParameter(name='idfigure', value='1', format='int')
1277 opObj11.addParameter(name='wintitle', value='SpectraPlot0', format='str')
1278 opObj11.addParameter(name='zmin', value='40', format='int')
1279 opObj11.addParameter(name='zmax', value='90', format='int')
1280 opObj11.addParameter(name='showprofile', value='1', format='int')
1281
1282 print "Escribiendo el archivo XML"
1283
1284 controllerObj.writeXml(filename)
1285
1286 print "Leyendo el archivo XML"
1287 controllerObj.readXml(filename)
1288 #controllerObj.printattr()
1289
1290 controllerObj.createObjects()
1291 controllerObj.connectObjects()
1292 controllerObj.run()
1293
1294 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1 +1,1
1 <Project description="Claire" id="002" name="script02"><ReadUnit datatype="VoltageReader" id="21" inputId="0" name="VoltageReader"><Operation id="211" name="run" priority="1" type="self"><Parameter format="str" id="2111" name="datatype" value="VoltageReader" /><Parameter format="str" id="2112" name="path" value="/media/nanosat/0BDE10E00BDE10E0/CLAIRE" /><Parameter format="date" id="2113" name="startDate" value="2017/07/26" /><Parameter format="date" id="2114" name="endDate" value="2017/07/26" /><Parameter format="time" id="2115" name="startTime" value="15:00:00" /><Parameter format="time" id="2116" name="endTime" value="16:00:00" /><Parameter format="int" id="2118" name="delay" value="30" /><Parameter format="int" id="2119" name="walk" value="1" /><Parameter format="int" id="2120" name="online" value="0" /></Operation><Operation id="212" name="printNumberOfBlock" priority="2" type="self" /></ReadUnit><ProcUnit datatype="VoltageProc" id="22" inputId="21" name="VoltageProc"><Operation id="221" name="run" priority="1" type="self" /></ProcUnit><ProcUnit datatype="SpectraProc" id="23" inputId="22" name="SpectraProc"><Operation id="231" name="run" priority="1" type="self"><Parameter format="int" id="2311" name="nFFTPoints" value="128" /><Parameter format="int" id="2312" name="nProfiles" value="128" /><Parameter format="pairslist" id="2313" name="pairsList" value="(0,1),(0,2),(1,2)" /></Operation><Operation id="232" name="setRadarFrequency" priority="2" type="self"><Parameter format="float" id="2321" name="frequency" value="445000000.0" /></Operation><Operation id="233" name="IncohInt" priority="3" type="other"><Parameter format="float" id="2331" name="timeInterval" value="128" /></Operation><Operation id="234" name="removeDC" priority="4" type="self"><Parameter format="int" id="2341" name="mode" value="2" /></Operation><Operation id="235" name="SpectraPlot" priority="5" type="external"><Parameter format="int" id="2351" name="id" value="235" /><Parameter format="str" id="2352" name="wintitle" value="SpectraPlot" /><Parameter format="str" id="2353" name="xaxis" value="frequency" /><Parameter format="float" id="2354" name="zmin" value="15.0" /><Parameter format="float" id="2355" name="zmax" value="50.0" /><Parameter format="int" id="2356" name="exp_code" value="21" /><Parameter format="int" id="2357" name="ftp_wei" value="0" /><Parameter format="int" id="2358" name="plot_pos" value="0" /></Operation></ProcUnit></Project> No newline at end of file
1 <Project description="Claire" id="002" name="script02"><ReadUnit datatype="VoltageReader" id="21" inputId="0" name="VoltageReader"><Operation id="211" name="run" priority="1" type="self"><Parameter format="str" id="2111" name="datatype" value="VoltageReader" /><Parameter format="str" id="2112" name="path" value="/media/nanosat/0BDE10E00BDE10E0/CLAIRE" /><Parameter format="date" id="2113" name="startDate" value="2017/07/26" /><Parameter format="date" id="2114" name="endDate" value="2017/07/26" /><Parameter format="time" id="2115" name="startTime" value="15:00:00" /><Parameter format="time" id="2116" name="endTime" value="16:00:00" /><Parameter format="int" id="2117" name="delay" value="30" /><Parameter format="int" id="2118" name="walk" value="1" /><Parameter format="int" id="2119" name="online" value="0" /></Operation><Operation id="212" name="printNumberOfBlock" priority="2" type="self" /></ReadUnit><ProcUnit datatype="VoltageProc" id="22" inputId="21" name="VoltageProc"><Operation id="221" name="run" priority="1" type="self" /></ProcUnit><ProcUnit datatype="SpectraProc" id="23" inputId="22" name="SpectraProc"><Operation id="231" name="run" priority="1" type="self"><Parameter format="int" id="2311" name="nFFTPoints" value="128" /><Parameter format="int" id="2312" name="nProfiles" value="128" /><Parameter format="pairslist" id="2313" name="pairsList" value="(0,1),(0,2),(1,2)" /></Operation><Operation id="232" name="setRadarFrequency" priority="2" type="self"><Parameter format="float" id="2321" name="frequency" value="445000000.0" /></Operation><Operation id="233" name="IncohInt" priority="3" type="other"><Parameter format="float" id="2331" name="timeInterval" value="128" /></Operation><Operation id="234" name="removeDC" priority="4" type="self"><Parameter format="int" id="2341" name="mode" value="2" /></Operation><Operation id="235" name="SpectraPlot" priority="5" type="external"><Parameter format="int" id="2351" name="id" value="235" /><Parameter format="str" id="2352" name="wintitle" value="SpectraPlot" /><Parameter format="str" id="2353" name="xaxis" value="frequency" /><Parameter format="float" id="2354" name="zmin" value="15.0" /><Parameter format="float" id="2355" name="zmax" value="50.0" /><Parameter format="int" id="2356" name="exp_code" value="21" /><Parameter format="int" id="2357" name="ftp_wei" value="0" /><Parameter format="int" id="2358" name="plot_pos" value="0" /></Operation></ProcUnit></Project> No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now