##// END OF EJS Templates
schain
RSS
formatting, bug en controller.py
José Chávez -
r1017:934ca8cba82f
parent child
Show More
Side by Side Unified
Context file:
r1017:934ca8cba82f Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1333 +1,1328
1 '''
1 '''
2 Created on September , 2012
2 Created on September , 2012
3 @author:
3 @author:
4 '''
4 '''
5
5
6 import sys
6 import sys
7 import ast
7 import ast
8 import datetime
8 import datetime
9 import traceback
9 import traceback
10 import math
10 import math
11 import time
11 import time
12 from multiprocessing import Process, Queue, cpu_count
12 from multiprocessing import Process, Queue, cpu_count
13 from profilehooks import profile, coverage
14
13
15 import schainpy
14 import schainpy
16 import schainpy.admin
15 import schainpy.admin
17
16
18 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
17 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
19 from xml.dom import minidom
18 from xml.dom import minidom
20
19
21 from schainpy.model import *
20 from schainpy.model import *
22 from time import sleep
21 from time import sleep
23
22
24
23
25
24
26 def prettify(elem):
25 def prettify(elem):
27 """Return a pretty-printed XML string for the Element.
26 """Return a pretty-printed XML string for the Element.
28 """
27 """
29 rough_string = tostring(elem, 'utf-8')
28 rough_string = tostring(elem, 'utf-8')
30 reparsed = minidom.parseString(rough_string)
29 reparsed = minidom.parseString(rough_string)
31 return reparsed.toprettyxml(indent=" ")
30 return reparsed.toprettyxml(indent=" ")
32
31
33 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
32 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
34 skip = 0
33 skip = 0
35 cursor = 0
34 cursor = 0
36 nFiles = None
35 nFiles = None
37 processes = []
36 processes = []
38 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
37 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
39 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
38 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
40 days = (dt2 - dt1).days
39 days = (dt2 - dt1).days
41
40
42 for day in range(days+1):
41 for day in range(days+1):
43 skip = 0
42 skip = 0
44 cursor = 0
43 cursor = 0
45 q = Queue()
44 q = Queue()
46 processes = []
45 processes = []
47 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
46 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
48 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
47 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
49 firstProcess.start()
48 firstProcess.start()
50 if by_day:
49 if by_day:
51 continue
50 continue
52 nFiles = q.get()
51 nFiles = q.get()
53 if nFiles==0:
52 if nFiles==0:
54 continue
53 continue
55 firstProcess.terminate()
54 firstProcess.terminate()
56 skip = int(math.ceil(nFiles/nProcess))
55 skip = int(math.ceil(nFiles/nProcess))
57 while True:
56 while True:
58 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
57 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
59 processes[cursor].start()
58 processes[cursor].start()
60 if nFiles < cursor*skip:
59 if nFiles < cursor*skip:
61 break
60 break
62 cursor += 1
61 cursor += 1
63
62
64 def beforeExit(exctype, value, trace):
63 def beforeExit(exctype, value, trace):
65 for process in processes:
64 for process in processes:
66 process.terminate()
65 process.terminate()
67 process.join()
66 process.join()
68 print traceback.print_tb(trace)
67 print traceback.print_tb(trace)
69
68
70 sys.excepthook = beforeExit
69 sys.excepthook = beforeExit
71
70
72 for process in processes:
71 for process in processes:
73 process.join()
72 process.join()
74 process.terminate()
73 process.terminate()
75
74
76 time.sleep(3)
75 time.sleep(3)
77
76
78
77
79 class ParameterConf():
78 class ParameterConf():
80
79
81 id = None
80 id = None
82 name = None
81 name = None
83 value = None
82 value = None
84 format = None
83 format = None
85
84
86 __formated_value = None
85 __formated_value = None
87
86
88 ELEMENTNAME = 'Parameter'
87 ELEMENTNAME = 'Parameter'
89
88
90 def __init__(self):
89 def __init__(self):
91
90
92 self.format = 'str'
91 self.format = 'str'
93
92
94 def getElementName(self):
93 def getElementName(self):
95
94
96 return self.ELEMENTNAME
95 return self.ELEMENTNAME
97
96
98 def getValue(self):
97 def getValue(self):
99
98
100 value = self.value
99 value = self.value
101 format = self.format
100 format = self.format
102
101
103 if self.__formated_value != None:
102 if self.__formated_value != None:
104
103
105 return self.__formated_value
104 return self.__formated_value
106
105
107 if format == 'obj':
106 if format == 'obj':
108 return value
107 return value
109
108
110 if format == 'str':
109 if format == 'str':
111 self.__formated_value = str(value)
110 self.__formated_value = str(value)
112 return self.__formated_value
111 return self.__formated_value
113
112
114 if value == '':
113 if value == '':
115 raise ValueError, "%s: This parameter value is empty" %self.name
114 raise ValueError, "%s: This parameter value is empty" %self.name
116
115
117 if format == 'list':
116 if format == 'list':
118 strList = value.split(',')
117 strList = value.split(',')
119
118
120 self.__formated_value = strList
119 self.__formated_value = strList
121
120
122 return self.__formated_value
121 return self.__formated_value
123
122
124 if format == 'intlist':
123 if format == 'intlist':
125 """
124 """
126 Example:
125 Example:
127 value = (0,1,2)
126 value = (0,1,2)
128 """
127 """
129
128
130 new_value = ast.literal_eval(value)
129 new_value = ast.literal_eval(value)
131
130
132 if type(new_value) not in (tuple, list):
131 if type(new_value) not in (tuple, list):
133 new_value = [int(new_value)]
132 new_value = [int(new_value)]
134
133
135 self.__formated_value = new_value
134 self.__formated_value = new_value
136
135
137 return self.__formated_value
136 return self.__formated_value
138
137
139 if format == 'floatlist':
138 if format == 'floatlist':
140 """
139 """
141 Example:
140 Example:
142 value = (0.5, 1.4, 2.7)
141 value = (0.5, 1.4, 2.7)
143 """
142 """
144
143
145 new_value = ast.literal_eval(value)
144 new_value = ast.literal_eval(value)
146
145
147 if type(new_value) not in (tuple, list):
146 if type(new_value) not in (tuple, list):
148 new_value = [float(new_value)]
147 new_value = [float(new_value)]
149
148
150 self.__formated_value = new_value
149 self.__formated_value = new_value
151
150
152 return self.__formated_value
151 return self.__formated_value
153
152
154 if format == 'date':
153 if format == 'date':
155 strList = value.split('/')
154 strList = value.split('/')
156 intList = [int(x) for x in strList]
155 intList = [int(x) for x in strList]
157 date = datetime.date(intList[0], intList[1], intList[2])
156 date = datetime.date(intList[0], intList[1], intList[2])
158
157
159 self.__formated_value = date
158 self.__formated_value = date
160
159
161 return self.__formated_value
160 return self.__formated_value
162
161
163 if format == 'time':
162 if format == 'time':
164 strList = value.split(':')
163 strList = value.split(':')
165 intList = [int(x) for x in strList]
164 intList = [int(x) for x in strList]
166 time = datetime.time(intList[0], intList[1], intList[2])
165 time = datetime.time(intList[0], intList[1], intList[2])
167
166
168 self.__formated_value = time
167 self.__formated_value = time
169
168
170 return self.__formated_value
169 return self.__formated_value
171
170
172 if format == 'pairslist':
171 if format == 'pairslist':
173 """
172 """
174 Example:
173 Example:
175 value = (0,1),(1,2)
174 value = (0,1),(1,2)
176 """
175 """
177
176
178 new_value = ast.literal_eval(value)
177 new_value = ast.literal_eval(value)
179
178
180 if type(new_value) not in (tuple, list):
179 if type(new_value) not in (tuple, list):
181 raise ValueError, "%s has to be a tuple or list of pairs" %value
180 raise ValueError, "%s has to be a tuple or list of pairs" %value
182
181
183 if type(new_value[0]) not in (tuple, list):
182 if type(new_value[0]) not in (tuple, list):
184 if len(new_value) != 2:
183 if len(new_value) != 2:
185 raise ValueError, "%s has to be a tuple or list of pairs" %value
184 raise ValueError, "%s has to be a tuple or list of pairs" %value
186 new_value = [new_value]
185 new_value = [new_value]
187
186
188 for thisPair in new_value:
187 for thisPair in new_value:
189 if len(thisPair) != 2:
188 if len(thisPair) != 2:
190 raise ValueError, "%s has to be a tuple or list of pairs" %value
189 raise ValueError, "%s has to be a tuple or list of pairs" %value
191
190
192 self.__formated_value = new_value
191 self.__formated_value = new_value
193
192
194 return self.__formated_value
193 return self.__formated_value
195
194
196 if format == 'multilist':
195 if format == 'multilist':
197 """
196 """
198 Example:
197 Example:
199 value = (0,1,2),(3,4,5)
198 value = (0,1,2),(3,4,5)
200 """
199 """
201 multiList = ast.literal_eval(value)
200 multiList = ast.literal_eval(value)
202
201
203 if type(multiList[0]) == int:
202 if type(multiList[0]) == int:
204 multiList = ast.literal_eval("(" + value + ")")
203 multiList = ast.literal_eval("(" + value + ")")
205
204
206 self.__formated_value = multiList
205 self.__formated_value = multiList
207
206
208 return self.__formated_value
207 return self.__formated_value
209
208
210 if format == 'bool':
209 if format == 'bool':
211 value = int(value)
210 value = int(value)
212
211
213 if format == 'int':
212 if format == 'int':
214 value = float(value)
213 value = float(value)
215
214
216 format_func = eval(format)
215 format_func = eval(format)
217
216
218 self.__formated_value = format_func(value)
217 self.__formated_value = format_func(value)
219
218
220 return self.__formated_value
219 return self.__formated_value
221
220
222 def updateId(self, new_id):
221 def updateId(self, new_id):
223
222
224 self.id = str(new_id)
223 self.id = str(new_id)
225
224
226 def setup(self, id, name, value, format='str'):
225 def setup(self, id, name, value, format='str'):
227 self.id = str(id)
226 self.id = str(id)
228 self.name = name
227 self.name = name
229 if format == 'obj':
228 if format == 'obj':
230 self.value = value
229 self.value = value
231 else:
230 else:
232 self.value = str(value)
231 self.value = str(value)
233 self.format = str.lower(format)
232 self.format = str.lower(format)
234
233
235 self.getValue()
234 self.getValue()
236
235
237 return 1
236 return 1
238
237
239 def update(self, name, value, format='str'):
238 def update(self, name, value, format='str'):
240
239
241 self.name = name
240 self.name = name
242 self.value = str(value)
241 self.value = str(value)
243 self.format = format
242 self.format = format
244
243
245 def makeXml(self, opElement):
244 def makeXml(self, opElement):
246 if self.name not in ('queue',):
245 if self.name not in ('queue',):
247 parmElement = SubElement(opElement, self.ELEMENTNAME)
246 parmElement = SubElement(opElement, self.ELEMENTNAME)
248 parmElement.set('id', str(self.id))
247 parmElement.set('id', str(self.id))
249 parmElement.set('name', self.name)
248 parmElement.set('name', self.name)
250 parmElement.set('value', self.value)
249 parmElement.set('value', self.value)
251 parmElement.set('format', self.format)
250 parmElement.set('format', self.format)
252
251
253 def readXml(self, parmElement):
252 def readXml(self, parmElement):
254
253
255 self.id = parmElement.get('id')
254 self.id = parmElement.get('id')
256 self.name = parmElement.get('name')
255 self.name = parmElement.get('name')
257 self.value = parmElement.get('value')
256 self.value = parmElement.get('value')
258 self.format = str.lower(parmElement.get('format'))
257 self.format = str.lower(parmElement.get('format'))
259
258
260 #Compatible with old signal chain version
259 #Compatible with old signal chain version
261 if self.format == 'int' and self.name == 'idfigure':
260 if self.format == 'int' and self.name == 'idfigure':
262 self.name = 'id'
261 self.name = 'id'
263
262
264 def printattr(self):
263 def printattr(self):
265
264
266 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
265 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
267
266
268 class OperationConf():
267 class OperationConf():
269
268
270 id = None
269 id = None
271 name = None
270 name = None
272 priority = None
271 priority = None
273 type = None
272 type = None
274
273
275 parmConfObjList = []
274 parmConfObjList = []
276
275
277 ELEMENTNAME = 'Operation'
276 ELEMENTNAME = 'Operation'
278
277
279 def __init__(self):
278 def __init__(self):
280
279
281 self.id = '0'
280 self.id = '0'
282 self.name = None
281 self.name = None
283 self.priority = None
282 self.priority = None
284 self.type = 'self'
283 self.type = 'self'
285
284
286
285
287 def __getNewId(self):
286 def __getNewId(self):
288
287
289 return int(self.id)*10 + len(self.parmConfObjList) + 1
288 return int(self.id)*10 + len(self.parmConfObjList) + 1
290
289
291 def updateId(self, new_id):
290 def updateId(self, new_id):
292
291
293 self.id = str(new_id)
292 self.id = str(new_id)
294
293
295 n = 1
294 n = 1
296 for parmObj in self.parmConfObjList:
295 for parmObj in self.parmConfObjList:
297
296
298 idParm = str(int(new_id)*10 + n)
297 idParm = str(int(new_id)*10 + n)
299 parmObj.updateId(idParm)
298 parmObj.updateId(idParm)
300
299
301 n += 1
300 n += 1
302
301
303 def getElementName(self):
302 def getElementName(self):
304
303
305 return self.ELEMENTNAME
304 return self.ELEMENTNAME
306
305
307 def getParameterObjList(self):
306 def getParameterObjList(self):
308
307
309 return self.parmConfObjList
308 return self.parmConfObjList
310
309
311 def getParameterObj(self, parameterName):
310 def getParameterObj(self, parameterName):
312
311
313 for parmConfObj in self.parmConfObjList:
312 for parmConfObj in self.parmConfObjList:
314
313
315 if parmConfObj.name != parameterName:
314 if parmConfObj.name != parameterName:
316 continue
315 continue
317
316
318 return parmConfObj
317 return parmConfObj
319
318
320 return None
319 return None
321
320
322 def getParameterObjfromValue(self, parameterValue):
321 def getParameterObjfromValue(self, parameterValue):
323
322
324 for parmConfObj in self.parmConfObjList:
323 for parmConfObj in self.parmConfObjList:
325
324
326 if parmConfObj.getValue() != parameterValue:
325 if parmConfObj.getValue() != parameterValue:
327 continue
326 continue
328
327
329 return parmConfObj.getValue()
328 return parmConfObj.getValue()
330
329
331 return None
330 return None
332
331
333 def getParameterValue(self, parameterName):
332 def getParameterValue(self, parameterName):
334
333
335 parameterObj = self.getParameterObj(parameterName)
334 parameterObj = self.getParameterObj(parameterName)
336
335
337 # if not parameterObj:
336 # if not parameterObj:
338 # return None
337 # return None
339
338
340 value = parameterObj.getValue()
339 value = parameterObj.getValue()
341
340
342 return value
341 return value
343
342
344
343
345 def getKwargs(self):
344 def getKwargs(self):
346
345
347 kwargs = {}
346 kwargs = {}
348
347
349 for parmConfObj in self.parmConfObjList:
348 for parmConfObj in self.parmConfObjList:
350 if self.name == 'run' and parmConfObj.name == 'datatype':
349 if self.name == 'run' and parmConfObj.name == 'datatype':
351 continue
350 continue
352
351
353 kwargs[parmConfObj.name] = parmConfObj.getValue()
352 kwargs[parmConfObj.name] = parmConfObj.getValue()
354
353
355 return kwargs
354 return kwargs
356
355
357 def setup(self, id, name, priority, type):
356 def setup(self, id, name, priority, type):
358
357
359 self.id = str(id)
358 self.id = str(id)
360 self.name = name
359 self.name = name
361 self.type = type
360 self.type = type
362 self.priority = priority
361 self.priority = priority
363
362
364 self.parmConfObjList = []
363 self.parmConfObjList = []
365
364
366 def removeParameters(self):
365 def removeParameters(self):
367
366
368 for obj in self.parmConfObjList:
367 for obj in self.parmConfObjList:
369 del obj
368 del obj
370
369
371 self.parmConfObjList = []
370 self.parmConfObjList = []
372
371
373 def addParameter(self, name, value, format='str'):
372 def addParameter(self, name, value, format='str'):
374
373
375 id = self.__getNewId()
374 id = self.__getNewId()
376
375
377 parmConfObj = ParameterConf()
376 parmConfObj = ParameterConf()
378 if not parmConfObj.setup(id, name, value, format):
377 if not parmConfObj.setup(id, name, value, format):
379 return None
378 return None
380
379
381 self.parmConfObjList.append(parmConfObj)
380 self.parmConfObjList.append(parmConfObj)
382
381
383 return parmConfObj
382 return parmConfObj
384
383
385 def changeParameter(self, name, value, format='str'):
384 def changeParameter(self, name, value, format='str'):
386
385
387 parmConfObj = self.getParameterObj(name)
386 parmConfObj = self.getParameterObj(name)
388 parmConfObj.update(name, value, format)
387 parmConfObj.update(name, value, format)
389
388
390 return parmConfObj
389 return parmConfObj
391
390
392 def makeXml(self, procUnitElement):
391 def makeXml(self, procUnitElement):
393
392
394 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
393 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
395 opElement.set('id', str(self.id))
394 opElement.set('id', str(self.id))
396 opElement.set('name', self.name)
395 opElement.set('name', self.name)
397 opElement.set('type', self.type)
396 opElement.set('type', self.type)
398 opElement.set('priority', str(self.priority))
397 opElement.set('priority', str(self.priority))
399
398
400 for parmConfObj in self.parmConfObjList:
399 for parmConfObj in self.parmConfObjList:
401 parmConfObj.makeXml(opElement)
400 parmConfObj.makeXml(opElement)
402
401
403 def readXml(self, opElement):
402 def readXml(self, opElement):
404
403
405 self.id = opElement.get('id')
404 self.id = opElement.get('id')
406 self.name = opElement.get('name')
405 self.name = opElement.get('name')
407 self.type = opElement.get('type')
406 self.type = opElement.get('type')
408 self.priority = opElement.get('priority')
407 self.priority = opElement.get('priority')
409
408
410 #Compatible with old signal chain version
409 #Compatible with old signal chain version
411 #Use of 'run' method instead 'init'
410 #Use of 'run' method instead 'init'
412 if self.type == 'self' and self.name == 'init':
411 if self.type == 'self' and self.name == 'init':
413 self.name = 'run'
412 self.name = 'run'
414
413
415 self.parmConfObjList = []
414 self.parmConfObjList = []
416
415
417 parmElementList = opElement.iter(ParameterConf().getElementName())
416 parmElementList = opElement.iter(ParameterConf().getElementName())
418
417
419 for parmElement in parmElementList:
418 for parmElement in parmElementList:
420 parmConfObj = ParameterConf()
419 parmConfObj = ParameterConf()
421 parmConfObj.readXml(parmElement)
420 parmConfObj.readXml(parmElement)
422
421
423 #Compatible with old signal chain version
422 #Compatible with old signal chain version
424 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
423 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
425 if self.type != 'self' and self.name == 'Plot':
424 if self.type != 'self' and self.name == 'Plot':
426 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
425 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
427 self.name = parmConfObj.value
426 self.name = parmConfObj.value
428 continue
427 continue
429
428
430 self.parmConfObjList.append(parmConfObj)
429 self.parmConfObjList.append(parmConfObj)
431
430
432 def printattr(self):
431 def printattr(self):
433
432
434 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
433 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
435 self.id,
434 self.id,
436 self.name,
435 self.name,
437 self.type,
436 self.type,
438 self.priority)
437 self.priority)
439
438
440 for parmConfObj in self.parmConfObjList:
439 for parmConfObj in self.parmConfObjList:
441 parmConfObj.printattr()
440 parmConfObj.printattr()
442
441
443 def createObject(self, plotter_queue=None):
442 def createObject(self, plotter_queue=None):
444
443
445
444
446 if self.type == 'self':
445 if self.type == 'self':
447 raise ValueError, "This operation type cannot be created"
446 raise ValueError, "This operation type cannot be created"
448
447
449 if self.type == 'plotter':
448 if self.type == 'plotter':
450 #Plotter(plotter_name)
449 #Plotter(plotter_name)
451 if not plotter_queue:
450 if not plotter_queue:
452 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
451 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
453
452
454 opObj = Plotter(self.name, plotter_queue)
453 opObj = Plotter(self.name, plotter_queue)
455
454
456 if self.type == 'external' or self.type == 'other':
455 if self.type == 'external' or self.type == 'other':
457
456
458 className = eval(self.name)
457 className = eval(self.name)
459 kwargs = self.getKwargs()
458 kwargs = self.getKwargs()
460
459
461 opObj = className(**kwargs)
460 opObj = className(**kwargs)
462
461
463 return opObj
462 return opObj
464
463
465
464
466 class ProcUnitConf():
465 class ProcUnitConf():
467
466
468 id = None
467 id = None
469 name = None
468 name = None
470 datatype = None
469 datatype = None
471 inputId = None
470 inputId = None
472 parentId = None
471 parentId = None
473
472
474 opConfObjList = []
473 opConfObjList = []
475
474
476 procUnitObj = None
475 procUnitObj = None
477 opObjList = []
476 opObjList = []
478
477
479 ELEMENTNAME = 'ProcUnit'
478 ELEMENTNAME = 'ProcUnit'
480
479
481 def __init__(self):
480 def __init__(self):
482
481
483 self.id = None
482 self.id = None
484 self.datatype = None
483 self.datatype = None
485 self.name = None
484 self.name = None
486 self.inputId = None
485 self.inputId = None
487
486
488 self.opConfObjList = []
487 self.opConfObjList = []
489
488
490 self.procUnitObj = None
489 self.procUnitObj = None
491 self.opObjDict = {}
490 self.opObjDict = {}
492
491
493 def __getPriority(self):
492 def __getPriority(self):
494
493
495 return len(self.opConfObjList)+1
494 return len(self.opConfObjList)+1
496
495
497 def __getNewId(self):
496 def __getNewId(self):
498
497
499 return int(self.id)*10 + len(self.opConfObjList) + 1
498 return int(self.id)*10 + len(self.opConfObjList) + 1
500
499
501 def getElementName(self):
500 def getElementName(self):
502
501
503 return self.ELEMENTNAME
502 return self.ELEMENTNAME
504
503
505 def getId(self):
504 def getId(self):
506
505
507 return self.id
506 return self.id
508
507
509 def updateId(self, new_id, parentId=parentId):
508 def updateId(self, new_id, parentId=parentId):
510
509
511
510
512 new_id = int(parentId)*10 + (int(self.id) % 10)
511 new_id = int(parentId)*10 + (int(self.id) % 10)
513 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
512 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
514
513
515 #If this proc unit has not inputs
514 #If this proc unit has not inputs
516 if self.inputId == '0':
515 if self.inputId == '0':
517 new_inputId = 0
516 new_inputId = 0
518
517
519 n = 1
518 n = 1
520 for opConfObj in self.opConfObjList:
519 for opConfObj in self.opConfObjList:
521
520
522 idOp = str(int(new_id)*10 + n)
521 idOp = str(int(new_id)*10 + n)
523 opConfObj.updateId(idOp)
522 opConfObj.updateId(idOp)
524
523
525 n += 1
524 n += 1
526
525
527 self.parentId = str(parentId)
526 self.parentId = str(parentId)
528 self.id = str(new_id)
527 self.id = str(new_id)
529 self.inputId = str(new_inputId)
528 self.inputId = str(new_inputId)
530
529
531
530
532 def getInputId(self):
531 def getInputId(self):
533
532
534 return self.inputId
533 return self.inputId
535
534
536 def getOperationObjList(self):
535 def getOperationObjList(self):
537
536
538 return self.opConfObjList
537 return self.opConfObjList
539
538
540 def getOperationObj(self, name=None):
539 def getOperationObj(self, name=None):
541
540
542 for opConfObj in self.opConfObjList:
541 for opConfObj in self.opConfObjList:
543
542
544 if opConfObj.name != name:
543 if opConfObj.name != name:
545 continue
544 continue
546
545
547 return opConfObj
546 return opConfObj
548
547
549 return None
548 return None
550
549
551 def getOpObjfromParamValue(self, value=None):
550 def getOpObjfromParamValue(self, value=None):
552
551
553 for opConfObj in self.opConfObjList:
552 for opConfObj in self.opConfObjList:
554 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
553 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
555 continue
554 continue
556 return opConfObj
555 return opConfObj
557 return None
556 return None
558
557
559 def getProcUnitObj(self):
558 def getProcUnitObj(self):
560
559
561 return self.procUnitObj
560 return self.procUnitObj
562
561
563 def setup(self, id, name, datatype, inputId, parentId=None):
562 def setup(self, id, name, datatype, inputId, parentId=None):
564
563
565 #Compatible with old signal chain version
564 #Compatible with old signal chain version
566 if datatype==None and name==None:
565 if datatype==None and name==None:
567 raise ValueError, "datatype or name should be defined"
566 raise ValueError, "datatype or name should be defined"
568
567
569 if name==None:
568 if name==None:
570 if 'Proc' in datatype:
569 if 'Proc' in datatype:
571 name = datatype
570 name = datatype
572 else:
571 else:
573 name = '%sProc' %(datatype)
572 name = '%sProc' %(datatype)
574
573
575 if datatype==None:
574 if datatype==None:
576 datatype = name.replace('Proc','')
575 datatype = name.replace('Proc','')
577
576
578 self.id = str(id)
577 self.id = str(id)
579 self.name = name
578 self.name = name
580 self.datatype = datatype
579 self.datatype = datatype
581 self.inputId = inputId
580 self.inputId = inputId
582 self.parentId = parentId
581 self.parentId = parentId
583
582
584 self.opConfObjList = []
583 self.opConfObjList = []
585
584
586 self.addOperation(name='run', optype='self')
585 self.addOperation(name='run', optype='self')
587
586
588 def removeOperations(self):
587 def removeOperations(self):
589
588
590 for obj in self.opConfObjList:
589 for obj in self.opConfObjList:
591 del obj
590 del obj
592
591
593 self.opConfObjList = []
592 self.opConfObjList = []
594 self.addOperation(name='run')
593 self.addOperation(name='run')
595
594
596 def addParameter(self, **kwargs):
595 def addParameter(self, **kwargs):
597 '''
596 '''
598 Add parameters to "run" operation
597 Add parameters to "run" operation
599 '''
598 '''
600 opObj = self.opConfObjList[0]
599 opObj = self.opConfObjList[0]
601
600
602 opObj.addParameter(**kwargs)
601 opObj.addParameter(**kwargs)
603
602
604 return opObj
603 return opObj
605
604
606 def addOperation(self, name, optype='self'):
605 def addOperation(self, name, optype='self'):
607
606
608 id = self.__getNewId()
607 id = self.__getNewId()
609 priority = self.__getPriority()
608 priority = self.__getPriority()
610
609
611 opConfObj = OperationConf()
610 opConfObj = OperationConf()
612 opConfObj.setup(id, name=name, priority=priority, type=optype)
611 opConfObj.setup(id, name=name, priority=priority, type=optype)
613
612
614 self.opConfObjList.append(opConfObj)
613 self.opConfObjList.append(opConfObj)
615
614
616 return opConfObj
615 return opConfObj
617
616
618 def makeXml(self, projectElement):
617 def makeXml(self, projectElement):
619
618
620 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
619 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
621 procUnitElement.set('id', str(self.id))
620 procUnitElement.set('id', str(self.id))
622 procUnitElement.set('name', self.name)
621 procUnitElement.set('name', self.name)
623 procUnitElement.set('datatype', self.datatype)
622 procUnitElement.set('datatype', self.datatype)
624 procUnitElement.set('inputId', str(self.inputId))
623 procUnitElement.set('inputId', str(self.inputId))
625
624
626 for opConfObj in self.opConfObjList:
625 for opConfObj in self.opConfObjList:
627 opConfObj.makeXml(procUnitElement)
626 opConfObj.makeXml(procUnitElement)
628
627
629 def readXml(self, upElement):
628 def readXml(self, upElement):
630
629
631 self.id = upElement.get('id')
630 self.id = upElement.get('id')
632 self.name = upElement.get('name')
631 self.name = upElement.get('name')
633 self.datatype = upElement.get('datatype')
632 self.datatype = upElement.get('datatype')
634 self.inputId = upElement.get('inputId')
633 self.inputId = upElement.get('inputId')
635
634
636 if self.ELEMENTNAME == "ReadUnit":
635 if self.ELEMENTNAME == "ReadUnit":
637 self.datatype = self.datatype.replace("Reader", "")
636 self.datatype = self.datatype.replace("Reader", "")
638
637
639 if self.ELEMENTNAME == "ProcUnit":
638 if self.ELEMENTNAME == "ProcUnit":
640 self.datatype = self.datatype.replace("Proc", "")
639 self.datatype = self.datatype.replace("Proc", "")
641
640
642 if self.inputId == 'None':
641 if self.inputId == 'None':
643 self.inputId = '0'
642 self.inputId = '0'
644
643
645 self.opConfObjList = []
644 self.opConfObjList = []
646
645
647 opElementList = upElement.iter(OperationConf().getElementName())
646 opElementList = upElement.iter(OperationConf().getElementName())
648
647
649 for opElement in opElementList:
648 for opElement in opElementList:
650 opConfObj = OperationConf()
649 opConfObj = OperationConf()
651 opConfObj.readXml(opElement)
650 opConfObj.readXml(opElement)
652 self.opConfObjList.append(opConfObj)
651 self.opConfObjList.append(opConfObj)
653
652
654 def printattr(self):
653 def printattr(self):
655
654
656 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
655 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
657 self.id,
656 self.id,
658 self.name,
657 self.name,
659 self.datatype,
658 self.datatype,
660 self.inputId)
659 self.inputId)
661
660
662 for opConfObj in self.opConfObjList:
661 for opConfObj in self.opConfObjList:
663 opConfObj.printattr()
662 opConfObj.printattr()
664
663
665
664
666 def getKwargs(self):
665 def getKwargs(self):
667
666
668 opObj = self.opConfObjList[0]
667 opObj = self.opConfObjList[0]
669 kwargs = opObj.getKwargs()
668 kwargs = opObj.getKwargs()
670
669
671 return kwargs
670 return kwargs
672
671
673 def createObjects(self, plotter_queue=None):
672 def createObjects(self, plotter_queue=None):
674
673
675 className = eval(self.name)
674 className = eval(self.name)
676 kwargs = self.getKwargs()
675 kwargs = self.getKwargs()
677 procUnitObj = className(**kwargs)
676 procUnitObj = className(**kwargs)
678
677
679 for opConfObj in self.opConfObjList:
678 for opConfObj in self.opConfObjList:
680
679
681 if opConfObj.type=='self' and self.name=='run':
680 if opConfObj.type=='self' and self.name=='run':
682 continue
681 continue
683 elif opConfObj.type=='self':
682 elif opConfObj.type=='self':
684 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
683 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
685 continue
684 continue
686
685
687 opObj = opConfObj.createObject(plotter_queue)
686 opObj = opConfObj.createObject(plotter_queue)
688
687
689 self.opObjDict[opConfObj.id] = opObj
688 self.opObjDict[opConfObj.id] = opObj
690
689
691 procUnitObj.addOperation(opObj, opConfObj.id)
690 procUnitObj.addOperation(opObj, opConfObj.id)
692
691
693 self.procUnitObj = procUnitObj
692 self.procUnitObj = procUnitObj
694
693
695 return procUnitObj
694 return procUnitObj
696
695
697 ## @profile
698 def run(self):
696 def run(self):
699
697
700 is_ok = False
698 is_ok = False
701
699
702 for opConfObj in self.opConfObjList:
700 for opConfObj in self.opConfObjList:
703
701
704 kwargs = {}
702 kwargs = {}
705 for parmConfObj in opConfObj.getParameterObjList():
703 for parmConfObj in opConfObj.getParameterObjList():
706 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
704 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
707 continue
705 continue
708
706
709 kwargs[parmConfObj.name] = parmConfObj.getValue()
707 kwargs[parmConfObj.name] = parmConfObj.getValue()
710
708
711 #ini = time.time()
709 #ini = time.time()
712
710
713 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
711 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
714 sts = self.procUnitObj.call(opType = opConfObj.type,
712 sts = self.procUnitObj.call(opType = opConfObj.type,
715 opName = opConfObj.name,
713 opName = opConfObj.name,
716 opId = opConfObj.id,
714 opId = opConfObj.id)
717 **kwargs)
718
715
719 # total_time = time.time() - ini
716 # total_time = time.time() - ini
720 #
717 #
721 # if total_time > 0.002:
718 # if total_time > 0.002:
722 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
719 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
723
720
724 is_ok = is_ok or sts
721 is_ok = is_ok or sts
725
722
726 return is_ok
723 return is_ok
727
724
728 def close(self):
725 def close(self):
729
726
730 for opConfObj in self.opConfObjList:
727 for opConfObj in self.opConfObjList:
731 if opConfObj.type == 'self':
728 if opConfObj.type == 'self':
732 continue
729 continue
733
730
734 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
731 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
735 opObj.close()
732 opObj.close()
736
733
737 self.procUnitObj.close()
734 self.procUnitObj.close()
738
735
739 return
736 return
740
737
741 class ReadUnitConf(ProcUnitConf):
738 class ReadUnitConf(ProcUnitConf):
742
739
743 path = None
740 path = None
744 startDate = None
741 startDate = None
745 endDate = None
742 endDate = None
746 startTime = None
743 startTime = None
747 endTime = None
744 endTime = None
748
745
749 ELEMENTNAME = 'ReadUnit'
746 ELEMENTNAME = 'ReadUnit'
750
747
751 def __init__(self):
748 def __init__(self):
752
749
753 self.id = None
750 self.id = None
754 self.datatype = None
751 self.datatype = None
755 self.name = None
752 self.name = None
756 self.inputId = None
753 self.inputId = None
757
754
758 self.parentId = None
755 self.parentId = None
759
756
760 self.opConfObjList = []
757 self.opConfObjList = []
761 self.opObjList = []
758 self.opObjList = []
762
759
763 def getElementName(self):
760 def getElementName(self):
764
761
765 return self.ELEMENTNAME
762 return self.ELEMENTNAME
766
763
767 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
764 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
768 endTime="", parentId=None, queue=None, server=None, **kwargs):
765 endTime="", parentId=None, queue=None, server=None, **kwargs):
769
770 #Compatible with old signal chain version
766 #Compatible with old signal chain version
771 if datatype==None and name==None:
767 if datatype==None and name==None:
772 raise ValueError, "datatype or name should be defined"
768 raise ValueError, "datatype or name should be defined"
773
769
774 if name==None:
770 if name==None:
775 if 'Reader' in datatype:
771 if 'Reader' in datatype:
776 name = datatype
772 name = datatype
777 else:
773 else:
778 name = '%sReader' %(datatype)
774 name = '%sReader' %(datatype)
779 if datatype==None:
775 if datatype==None:
780 datatype = name.replace('Reader','')
776 datatype = name.replace('Reader','')
781
777
782 self.id = id
778 self.id = id
783 self.name = name
779 self.name = name
784 self.datatype = datatype
780 self.datatype = datatype
785 if path != '':
781 if path != '':
786 self.path = os.path.abspath(path)
782 self.path = os.path.abspath(path)
787 self.startDate = startDate
783 self.startDate = startDate
788 self.endDate = endDate
784 self.endDate = endDate
789 self.startTime = startTime
785 self.startTime = startTime
790 self.endTime = endTime
786 self.endTime = endTime
791
787
792 self.inputId = '0'
788 self.inputId = '0'
793 self.parentId = parentId
789 self.parentId = parentId
794 self.queue = queue
790 self.queue = queue
795 self.server = server
791 self.server = server
796 self.addRunOperation(**kwargs)
792 self.addRunOperation(**kwargs)
797
793
798 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
794 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
799
795
800 #Compatible with old signal chain version
796 #Compatible with old signal chain version
801 if datatype==None and name==None:
797 if datatype==None and name==None:
802 raise ValueError, "datatype or name should be defined"
798 raise ValueError, "datatype or name should be defined"
803
799
804 if name==None:
800 if name==None:
805 if 'Reader' in datatype:
801 if 'Reader' in datatype:
806 name = datatype
802 name = datatype
807 else:
803 else:
808 name = '%sReader' %(datatype)
804 name = '%sReader' %(datatype)
809
805
810 if datatype==None:
806 if datatype==None:
811 datatype = name.replace('Reader','')
807 datatype = name.replace('Reader','')
812
808
813 self.datatype = datatype
809 self.datatype = datatype
814 self.name = name
810 self.name = name
815 self.path = path
811 self.path = path
816 self.startDate = startDate
812 self.startDate = startDate
817 self.endDate = endDate
813 self.endDate = endDate
818 self.startTime = startTime
814 self.startTime = startTime
819 self.endTime = endTime
815 self.endTime = endTime
820
816
821 self.inputId = '0'
817 self.inputId = '0'
822 self.parentId = parentId
818 self.parentId = parentId
823
819
824 self.updateRunOperation(**kwargs)
820 self.updateRunOperation(**kwargs)
825
821
826 def removeOperations(self):
822 def removeOperations(self):
827
823
828 for obj in self.opConfObjList:
824 for obj in self.opConfObjList:
829 del obj
825 del obj
830
826
831 self.opConfObjList = []
827 self.opConfObjList = []
832
828
833 def addRunOperation(self, **kwargs):
829 def addRunOperation(self, **kwargs):
834
830
835 opObj = self.addOperation(name = 'run', optype = 'self')
831 opObj = self.addOperation(name = 'run', optype = 'self')
836
832
837 if self.server is None:
833 if self.server is None:
838 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
834 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
839 opObj.addParameter(name='path' , value=self.path, format='str')
835 opObj.addParameter(name='path' , value=self.path, format='str')
840 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
836 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
841 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
837 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
842 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
838 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
843 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
839 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
844 opObj.addParameter(name='queue' , value=self.queue, format='obj')
840 opObj.addParameter(name='queue' , value=self.queue, format='obj')
845 for key, value in kwargs.items():
841 for key, value in kwargs.items():
846 opObj.addParameter(name=key, value=value, format=type(value).__name__)
842 opObj.addParameter(name=key, value=value, format=type(value).__name__)
847 else:
843 else:
848 opObj.addParameter(name='server' , value=self.server, format='str')
844 opObj.addParameter(name='server' , value=self.server, format='str')
849
845
850
846
851 return opObj
847 return opObj
852
848
853 def updateRunOperation(self, **kwargs):
849 def updateRunOperation(self, **kwargs):
854
850
855 opObj = self.getOperationObj(name = 'run')
851 opObj = self.getOperationObj(name = 'run')
856 opObj.removeParameters()
852 opObj.removeParameters()
857
853
858 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
854 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
859 opObj.addParameter(name='path' , value=self.path, format='str')
855 opObj.addParameter(name='path' , value=self.path, format='str')
860 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
856 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
861 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
857 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
862 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
858 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
863 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
859 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
864
860
865 for key, value in kwargs.items():
861 for key, value in kwargs.items():
866 opObj.addParameter(name=key, value=value, format=type(value).__name__)
862 opObj.addParameter(name=key, value=value, format=type(value).__name__)
867
863
868 return opObj
864 return opObj
869
865
870 # def makeXml(self, projectElement):
866 # def makeXml(self, projectElement):
871 #
867 #
872 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
868 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
873 # procUnitElement.set('id', str(self.id))
869 # procUnitElement.set('id', str(self.id))
874 # procUnitElement.set('name', self.name)
870 # procUnitElement.set('name', self.name)
875 # procUnitElement.set('datatype', self.datatype)
871 # procUnitElement.set('datatype', self.datatype)
876 # procUnitElement.set('inputId', str(self.inputId))
872 # procUnitElement.set('inputId', str(self.inputId))
877 #
873 #
878 # for opConfObj in self.opConfObjList:
874 # for opConfObj in self.opConfObjList:
879 # opConfObj.makeXml(procUnitElement)
875 # opConfObj.makeXml(procUnitElement)
880
876
881 def readXml(self, upElement):
877 def readXml(self, upElement):
882
878
883 self.id = upElement.get('id')
879 self.id = upElement.get('id')
884 self.name = upElement.get('name')
880 self.name = upElement.get('name')
885 self.datatype = upElement.get('datatype')
881 self.datatype = upElement.get('datatype')
886 self.inputId = upElement.get('inputId')
882 self.inputId = upElement.get('inputId')
887
883
888 if self.ELEMENTNAME == "ReadUnit":
884 if self.ELEMENTNAME == "ReadUnit":
889 self.datatype = self.datatype.replace("Reader", "")
885 self.datatype = self.datatype.replace("Reader", "")
890
886
891 if self.inputId == 'None':
887 if self.inputId == 'None':
892 self.inputId = '0'
888 self.inputId = '0'
893
889
894 self.opConfObjList = []
890 self.opConfObjList = []
895
891
896 opElementList = upElement.iter(OperationConf().getElementName())
892 opElementList = upElement.iter(OperationConf().getElementName())
897
893
898 for opElement in opElementList:
894 for opElement in opElementList:
899 opConfObj = OperationConf()
895 opConfObj = OperationConf()
900 opConfObj.readXml(opElement)
896 opConfObj.readXml(opElement)
901 self.opConfObjList.append(opConfObj)
897 self.opConfObjList.append(opConfObj)
902
898
903 if opConfObj.name == 'run':
899 if opConfObj.name == 'run':
904 self.path = opConfObj.getParameterValue('path')
900 self.path = opConfObj.getParameterValue('path')
905 self.startDate = opConfObj.getParameterValue('startDate')
901 self.startDate = opConfObj.getParameterValue('startDate')
906 self.endDate = opConfObj.getParameterValue('endDate')
902 self.endDate = opConfObj.getParameterValue('endDate')
907 self.startTime = opConfObj.getParameterValue('startTime')
903 self.startTime = opConfObj.getParameterValue('startTime')
908 self.endTime = opConfObj.getParameterValue('endTime')
904 self.endTime = opConfObj.getParameterValue('endTime')
909
905
910 class Project():
906 class Project():
911
907
912 id = None
908 id = None
913 name = None
909 name = None
914 description = None
910 description = None
915 filename = None
911 filename = None
916
912
917 procUnitConfObjDict = None
913 procUnitConfObjDict = None
918
914
919 ELEMENTNAME = 'Project'
915 ELEMENTNAME = 'Project'
920
916
921 plotterQueue = None
917 plotterQueue = None
922
918
923 def __init__(self, plotter_queue=None):
919 def __init__(self, plotter_queue=None):
924
920
925 self.id = None
921 self.id = None
926 self.name = None
922 self.name = None
927 self.description = None
923 self.description = None
928
924
929 self.plotterQueue = plotter_queue
925 self.plotterQueue = plotter_queue
930
926
931 self.procUnitConfObjDict = {}
927 self.procUnitConfObjDict = {}
932
928
933 def __getNewId(self):
929 def __getNewId(self):
934
930
935 idList = self.procUnitConfObjDict.keys()
931 idList = self.procUnitConfObjDict.keys()
936
932
937 id = int(self.id)*10
933 id = int(self.id)*10
938
934
939 while True:
935 while True:
940 id += 1
936 id += 1
941
937
942 if str(id) in idList:
938 if str(id) in idList:
943 continue
939 continue
944
940
945 break
941 break
946
942
947 return str(id)
943 return str(id)
948
944
949 def getElementName(self):
945 def getElementName(self):
950
946
951 return self.ELEMENTNAME
947 return self.ELEMENTNAME
952
948
953 def getId(self):
949 def getId(self):
954
950
955 return self.id
951 return self.id
956
952
957 def updateId(self, new_id):
953 def updateId(self, new_id):
958
954
959 self.id = str(new_id)
955 self.id = str(new_id)
960
956
961 keyList = self.procUnitConfObjDict.keys()
957 keyList = self.procUnitConfObjDict.keys()
962 keyList.sort()
958 keyList.sort()
963
959
964 n = 1
960 n = 1
965 newProcUnitConfObjDict = {}
961 newProcUnitConfObjDict = {}
966
962
967 for procKey in keyList:
963 for procKey in keyList:
968
964
969 procUnitConfObj = self.procUnitConfObjDict[procKey]
965 procUnitConfObj = self.procUnitConfObjDict[procKey]
970 idProcUnit = str(int(self.id)*10 + n)
966 idProcUnit = str(int(self.id)*10 + n)
971 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
967 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
972
968
973 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
969 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
974 n += 1
970 n += 1
975
971
976 self.procUnitConfObjDict = newProcUnitConfObjDict
972 self.procUnitConfObjDict = newProcUnitConfObjDict
977
973
978 def setup(self, id, name, description):
974 def setup(self, id, name, description):
979
975
980 self.id = str(id)
976 self.id = str(id)
981 self.name = name
977 self.name = name
982 self.description = description
978 self.description = description
983
979
984 def update(self, name, description):
980 def update(self, name, description):
985
981
986 self.name = name
982 self.name = name
987 self.description = description
983 self.description = description
988
984
989 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
985 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
990
991 if id is None:
986 if id is None:
992 idReadUnit = self.__getNewId()
987 idReadUnit = self.__getNewId()
993 else:
988 else:
994 idReadUnit = str(id)
989 idReadUnit = str(id)
995
990
996 readUnitConfObj = ReadUnitConf()
991 readUnitConfObj = ReadUnitConf()
997 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
992 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
998
993
999 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
994 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1000
995
1001 return readUnitConfObj
996 return readUnitConfObj
1002
997
1003 def addProcUnit(self, inputId='0', datatype=None, name=None):
998 def addProcUnit(self, inputId='0', datatype=None, name=None):
1004
999
1005 idProcUnit = self.__getNewId()
1000 idProcUnit = self.__getNewId()
1006
1001
1007 procUnitConfObj = ProcUnitConf()
1002 procUnitConfObj = ProcUnitConf()
1008 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1003 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1009
1004
1010 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1005 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1011
1006
1012 return procUnitConfObj
1007 return procUnitConfObj
1013
1008
1014 def removeProcUnit(self, id):
1009 def removeProcUnit(self, id):
1015
1010
1016 if id in self.procUnitConfObjDict.keys():
1011 if id in self.procUnitConfObjDict.keys():
1017 self.procUnitConfObjDict.pop(id)
1012 self.procUnitConfObjDict.pop(id)
1018
1013
1019 def getReadUnitId(self):
1014 def getReadUnitId(self):
1020
1015
1021 readUnitConfObj = self.getReadUnitObj()
1016 readUnitConfObj = self.getReadUnitObj()
1022
1017
1023 return readUnitConfObj.id
1018 return readUnitConfObj.id
1024
1019
1025 def getReadUnitObj(self):
1020 def getReadUnitObj(self):
1026
1021
1027 for obj in self.procUnitConfObjDict.values():
1022 for obj in self.procUnitConfObjDict.values():
1028 if obj.getElementName() == "ReadUnit":
1023 if obj.getElementName() == "ReadUnit":
1029 return obj
1024 return obj
1030
1025
1031 return None
1026 return None
1032
1027
1033 def getProcUnitObj(self, id=None, name=None):
1028 def getProcUnitObj(self, id=None, name=None):
1034
1029
1035 if id != None:
1030 if id != None:
1036 return self.procUnitConfObjDict[id]
1031 return self.procUnitConfObjDict[id]
1037
1032
1038 if name != None:
1033 if name != None:
1039 return self.getProcUnitObjByName(name)
1034 return self.getProcUnitObjByName(name)
1040
1035
1041 return None
1036 return None
1042
1037
1043 def getProcUnitObjByName(self, name):
1038 def getProcUnitObjByName(self, name):
1044
1039
1045 for obj in self.procUnitConfObjDict.values():
1040 for obj in self.procUnitConfObjDict.values():
1046 if obj.name == name:
1041 if obj.name == name:
1047 return obj
1042 return obj
1048
1043
1049 return None
1044 return None
1050
1045
1051 def procUnitItems(self):
1046 def procUnitItems(self):
1052
1047
1053 return self.procUnitConfObjDict.items()
1048 return self.procUnitConfObjDict.items()
1054
1049
1055 def makeXml(self):
1050 def makeXml(self):
1056
1051
1057 projectElement = Element('Project')
1052 projectElement = Element('Project')
1058 projectElement.set('id', str(self.id))
1053 projectElement.set('id', str(self.id))
1059 projectElement.set('name', self.name)
1054 projectElement.set('name', self.name)
1060 projectElement.set('description', self.description)
1055 projectElement.set('description', self.description)
1061
1056
1062 for procUnitConfObj in self.procUnitConfObjDict.values():
1057 for procUnitConfObj in self.procUnitConfObjDict.values():
1063 procUnitConfObj.makeXml(projectElement)
1058 procUnitConfObj.makeXml(projectElement)
1064
1059
1065 self.projectElement = projectElement
1060 self.projectElement = projectElement
1066
1061
1067 def writeXml(self, filename=None):
1062 def writeXml(self, filename=None):
1068
1063
1069 if filename == None:
1064 if filename == None:
1070 if self.filename:
1065 if self.filename:
1071 filename = self.filename
1066 filename = self.filename
1072 else:
1067 else:
1073 filename = "schain.xml"
1068 filename = "schain.xml"
1074
1069
1075 if not filename:
1070 if not filename:
1076 print "filename has not been defined. Use setFilename(filename) for do it."
1071 print "filename has not been defined. Use setFilename(filename) for do it."
1077 return 0
1072 return 0
1078
1073
1079 abs_file = os.path.abspath(filename)
1074 abs_file = os.path.abspath(filename)
1080
1075
1081 if not os.access(os.path.dirname(abs_file), os.W_OK):
1076 if not os.access(os.path.dirname(abs_file), os.W_OK):
1082 print "No write permission on %s" %os.path.dirname(abs_file)
1077 print "No write permission on %s" %os.path.dirname(abs_file)
1083 return 0
1078 return 0
1084
1079
1085 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1080 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1086 print "File %s already exists and it could not be overwriten" %abs_file
1081 print "File %s already exists and it could not be overwriten" %abs_file
1087 return 0
1082 return 0
1088
1083
1089 self.makeXml()
1084 self.makeXml()
1090
1085
1091 ElementTree(self.projectElement).write(abs_file, method='xml')
1086 ElementTree(self.projectElement).write(abs_file, method='xml')
1092
1087
1093 self.filename = abs_file
1088 self.filename = abs_file
1094
1089
1095 return 1
1090 return 1
1096
1091
1097 def readXml(self, filename = None):
1092 def readXml(self, filename = None):
1098
1093
1099 if not filename:
1094 if not filename:
1100 print "filename is not defined"
1095 print "filename is not defined"
1101 return 0
1096 return 0
1102
1097
1103 abs_file = os.path.abspath(filename)
1098 abs_file = os.path.abspath(filename)
1104
1099
1105 if not os.path.isfile(abs_file):
1100 if not os.path.isfile(abs_file):
1106 print "%s file does not exist" %abs_file
1101 print "%s file does not exist" %abs_file
1107 return 0
1102 return 0
1108
1103
1109 self.projectElement = None
1104 self.projectElement = None
1110 self.procUnitConfObjDict = {}
1105 self.procUnitConfObjDict = {}
1111
1106
1112 try:
1107 try:
1113 self.projectElement = ElementTree().parse(abs_file)
1108 self.projectElement = ElementTree().parse(abs_file)
1114 except:
1109 except:
1115 print "Error reading %s, verify file format" %filename
1110 print "Error reading %s, verify file format" %filename
1116 return 0
1111 return 0
1117
1112
1118 self.project = self.projectElement.tag
1113 self.project = self.projectElement.tag
1119
1114
1120 self.id = self.projectElement.get('id')
1115 self.id = self.projectElement.get('id')
1121 self.name = self.projectElement.get('name')
1116 self.name = self.projectElement.get('name')
1122 self.description = self.projectElement.get('description')
1117 self.description = self.projectElement.get('description')
1123
1118
1124 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1119 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1125
1120
1126 for readUnitElement in readUnitElementList:
1121 for readUnitElement in readUnitElementList:
1127 readUnitConfObj = ReadUnitConf()
1122 readUnitConfObj = ReadUnitConf()
1128 readUnitConfObj.readXml(readUnitElement)
1123 readUnitConfObj.readXml(readUnitElement)
1129
1124
1130 if readUnitConfObj.parentId == None:
1125 if readUnitConfObj.parentId == None:
1131 readUnitConfObj.parentId = self.id
1126 readUnitConfObj.parentId = self.id
1132
1127
1133 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1128 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1134
1129
1135 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1130 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1136
1131
1137 for procUnitElement in procUnitElementList:
1132 for procUnitElement in procUnitElementList:
1138 procUnitConfObj = ProcUnitConf()
1133 procUnitConfObj = ProcUnitConf()
1139 procUnitConfObj.readXml(procUnitElement)
1134 procUnitConfObj.readXml(procUnitElement)
1140
1135
1141 if procUnitConfObj.parentId == None:
1136 if procUnitConfObj.parentId == None:
1142 procUnitConfObj.parentId = self.id
1137 procUnitConfObj.parentId = self.id
1143
1138
1144 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1139 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1145
1140
1146 self.filename = abs_file
1141 self.filename = abs_file
1147
1142
1148 return 1
1143 return 1
1149
1144
1150 def printattr(self):
1145 def printattr(self):
1151
1146
1152 print "Project[%s]: name = %s, description = %s" %(self.id,
1147 print "Project[%s]: name = %s, description = %s" %(self.id,
1153 self.name,
1148 self.name,
1154 self.description)
1149 self.description)
1155
1150
1156 for procUnitConfObj in self.procUnitConfObjDict.values():
1151 for procUnitConfObj in self.procUnitConfObjDict.values():
1157 procUnitConfObj.printattr()
1152 procUnitConfObj.printattr()
1158
1153
1159 def createObjects(self):
1154 def createObjects(self):
1160
1155
1161 for procUnitConfObj in self.procUnitConfObjDict.values():
1156 for procUnitConfObj in self.procUnitConfObjDict.values():
1162 procUnitConfObj.createObjects(self.plotterQueue)
1157 procUnitConfObj.createObjects(self.plotterQueue)
1163
1158
1164 def __connect(self, objIN, thisObj):
1159 def __connect(self, objIN, thisObj):
1165
1160
1166 thisObj.setInput(objIN.getOutputObj())
1161 thisObj.setInput(objIN.getOutputObj())
1167
1162
1168 def connectObjects(self):
1163 def connectObjects(self):
1169
1164
1170 for thisPUConfObj in self.procUnitConfObjDict.values():
1165 for thisPUConfObj in self.procUnitConfObjDict.values():
1171
1166
1172 inputId = thisPUConfObj.getInputId()
1167 inputId = thisPUConfObj.getInputId()
1173
1168
1174 if int(inputId) == 0:
1169 if int(inputId) == 0:
1175 continue
1170 continue
1176
1171
1177 #Get input object
1172 #Get input object
1178 puConfINObj = self.procUnitConfObjDict[inputId]
1173 puConfINObj = self.procUnitConfObjDict[inputId]
1179 puObjIN = puConfINObj.getProcUnitObj()
1174 puObjIN = puConfINObj.getProcUnitObj()
1180
1175
1181 #Get current object
1176 #Get current object
1182 thisPUObj = thisPUConfObj.getProcUnitObj()
1177 thisPUObj = thisPUConfObj.getProcUnitObj()
1183
1178
1184 self.__connect(puObjIN, thisPUObj)
1179 self.__connect(puObjIN, thisPUObj)
1185
1180
1186 def __handleError(self, procUnitConfObj, send_email=True):
1181 def __handleError(self, procUnitConfObj, send_email=True):
1187
1182
1188 import socket
1183 import socket
1189
1184
1190 err = traceback.format_exception(sys.exc_info()[0],
1185 err = traceback.format_exception(sys.exc_info()[0],
1191 sys.exc_info()[1],
1186 sys.exc_info()[1],
1192 sys.exc_info()[2])
1187 sys.exc_info()[2])
1193
1188
1194 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1189 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1195 print "***** %s" %err[-1]
1190 print "***** %s" %err[-1]
1196
1191
1197 message = "".join(err)
1192 message = "".join(err)
1198
1193
1199 sys.stderr.write(message)
1194 sys.stderr.write(message)
1200
1195
1201 if not send_email:
1196 if not send_email:
1202 return
1197 return
1203
1198
1204 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1199 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1205
1200
1206 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1201 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1207 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1202 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1208 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1203 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1209 subtitle += "Configuration file: %s\n" %self.filename
1204 subtitle += "Configuration file: %s\n" %self.filename
1210 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1205 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1211
1206
1212 readUnitConfObj = self.getReadUnitObj()
1207 readUnitConfObj = self.getReadUnitObj()
1213 if readUnitConfObj:
1208 if readUnitConfObj:
1214 subtitle += "\nInput parameters:\n"
1209 subtitle += "\nInput parameters:\n"
1215 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1210 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1216 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1211 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1217 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1212 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1218 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1213 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1219 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1214 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1220 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1215 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1221
1216
1222 adminObj = schainpy.admin.SchainNotify()
1217 adminObj = schainpy.admin.SchainNotify()
1223 adminObj.sendAlert(message=message,
1218 adminObj.sendAlert(message=message,
1224 subject=subject,
1219 subject=subject,
1225 subtitle=subtitle,
1220 subtitle=subtitle,
1226 filename=self.filename)
1221 filename=self.filename)
1227
1222
1228 def isPaused(self):
1223 def isPaused(self):
1229 return 0
1224 return 0
1230
1225
1231 def isStopped(self):
1226 def isStopped(self):
1232 return 0
1227 return 0
1233
1228
1234 def runController(self):
1229 def runController(self):
1235 """
1230 """
1236 returns 0 when this process has been stopped, 1 otherwise
1231 returns 0 when this process has been stopped, 1 otherwise
1237 """
1232 """
1238
1233
1239 if self.isPaused():
1234 if self.isPaused():
1240 print "Process suspended"
1235 print "Process suspended"
1241
1236
1242 while True:
1237 while True:
1243 sleep(0.1)
1238 sleep(0.1)
1244
1239
1245 if not self.isPaused():
1240 if not self.isPaused():
1246 break
1241 break
1247
1242
1248 if self.isStopped():
1243 if self.isStopped():
1249 break
1244 break
1250
1245
1251 print "Process reinitialized"
1246 print "Process reinitialized"
1252
1247
1253 if self.isStopped():
1248 if self.isStopped():
1254 print "Process stopped"
1249 print "Process stopped"
1255 return 0
1250 return 0
1256
1251
1257 return 1
1252 return 1
1258
1253
1259 def setFilename(self, filename):
1254 def setFilename(self, filename):
1260
1255
1261 self.filename = filename
1256 self.filename = filename
1262
1257
1263 def setPlotterQueue(self, plotter_queue):
1258 def setPlotterQueue(self, plotter_queue):
1264
1259
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1260 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1266
1261
1267 def getPlotterQueue(self):
1262 def getPlotterQueue(self):
1268
1263
1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1264 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1270
1265
1271 def useExternalPlotter(self):
1266 def useExternalPlotter(self):
1272
1267
1273 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1268 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1274
1269
1275
1270
1276 def run(self):
1271 def run(self):
1277
1272
1278 print
1273 print
1279 print "*"*60
1274 print "*"*60
1280 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1275 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1281 print "*"*60
1276 print "*"*60
1282 print
1277 print
1283
1278
1284 keyList = self.procUnitConfObjDict.keys()
1279 keyList = self.procUnitConfObjDict.keys()
1285 keyList.sort()
1280 keyList.sort()
1286
1281
1287 while(True):
1282 while(True):
1288
1283
1289 is_ok = False
1284 is_ok = False
1290
1285
1291 for procKey in keyList:
1286 for procKey in keyList:
1292 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1287 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1293
1288
1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1289 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295
1290
1296 try:
1291 try:
1297 sts = procUnitConfObj.run()
1292 sts = procUnitConfObj.run()
1298 is_ok = is_ok or sts
1293 is_ok = is_ok or sts
1299 except KeyboardInterrupt:
1294 except KeyboardInterrupt:
1300 is_ok = False
1295 is_ok = False
1301 break
1296 break
1302 except ValueError, e:
1297 except ValueError, e:
1303 sleep(0.5)
1298 sleep(0.5)
1304 self.__handleError(procUnitConfObj, send_email=True)
1299 self.__handleError(procUnitConfObj, send_email=True)
1305 is_ok = False
1300 is_ok = False
1306 break
1301 break
1307 except:
1302 except:
1308 sleep(0.5)
1303 sleep(0.5)
1309 self.__handleError(procUnitConfObj)
1304 self.__handleError(procUnitConfObj)
1310 is_ok = False
1305 is_ok = False
1311 break
1306 break
1312
1307
1313 #If every process unit finished so end process
1308 #If every process unit finished so end process
1314 if not(is_ok):
1309 if not(is_ok):
1315 # print "Every process unit have finished"
1310 # print "Every process unit have finished"
1316 break
1311 break
1317
1312
1318 if not self.runController():
1313 if not self.runController():
1319 break
1314 break
1320
1315
1321 #Closing every process
1316 #Closing every process
1322 for procKey in keyList:
1317 for procKey in keyList:
1323 procUnitConfObj = self.procUnitConfObjDict[procKey]
1318 procUnitConfObj = self.procUnitConfObjDict[procKey]
1324 procUnitConfObj.close()
1319 procUnitConfObj.close()
1325
1320
1326 print "Process finished"
1321 print "Process finished"
1327
1322
1328 def start(self, filename=None):
1323 def start(self, filename=None):
1329
1324
1330 self.writeXml(filename)
1325 self.writeXml(filename)
1331 self.createObjects()
1326 self.createObjects()
1332 self.connectObjects()
1327 self.connectObjects()
1333 self.run()
1328 self.run()
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
Show file before | Show file after | Hide comments
@@ -1,349 +1,348
1 '''
1 '''
2
2
3 $Author: murco $
3 $Author: murco $
4 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
4 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
5 '''
5 '''
6 import inspect
6 import inspect
7 from fuzzywuzzy import process
7 from fuzzywuzzy import process
8
8
9 def checkKwargs(method, kwargs):
9 def checkKwargs(method, kwargs):
10 currentKwargs = kwargs
10 currentKwargs = kwargs
11 choices = inspect.getargspec(method).args
11 choices = inspect.getargspec(method).args
12 try:
12 try:
13 choices.remove('self')
13 choices.remove('self')
14 except Exception as e:
14 except Exception as e:
15 pass
15 pass
16
16
17 try:
17 try:
18 choices.remove('dataOut')
18 choices.remove('dataOut')
19 except Exception as e:
19 except Exception as e:
20 pass
20 pass
21
21
22 for kwarg in kwargs:
22 for kwarg in kwargs:
23 fuzz = process.extractOne(kwarg, choices)
23 fuzz = process.extractOne(kwarg, choices)
24 if fuzz is None:
24 if fuzz is None:
25 continue
25 continue
26 if fuzz[1] < 100:
26 if fuzz[1] < 100:
27 raise Exception('\x1b[0;32;40mDid you mean {} instead of {} in {}? \x1b[0m'.
27 raise Exception('\x1b[0;32;40mDid you mean {} instead of {} in {}? \x1b[0m'.
28 format(fuzz[0], kwarg, method.__self__.__class__.__name__))
28 format(fuzz[0], kwarg, method.__self__.__class__.__name__))
29
29
30 class ProcessingUnit(object):
30 class ProcessingUnit(object):
31
31
32 """
32 """
33 Esta es la clase base para el procesamiento de datos.
33 Esta es la clase base para el procesamiento de datos.
34
34
35 Contiene el metodo "call" para llamar operaciones. Las operaciones pueden ser:
35 Contiene el metodo "call" para llamar operaciones. Las operaciones pueden ser:
36 - Metodos internos (callMethod)
36 - Metodos internos (callMethod)
37 - Objetos del tipo Operation (callObject). Antes de ser llamados, estos objetos
37 - Objetos del tipo Operation (callObject). Antes de ser llamados, estos objetos
38 tienen que ser agreagados con el metodo "add".
38 tienen que ser agreagados con el metodo "add".
39
39
40 """
40 """
41 # objeto de datos de entrada (Voltage, Spectra o Correlation)
41 # objeto de datos de entrada (Voltage, Spectra o Correlation)
42 dataIn = None
42 dataIn = None
43 dataInList = []
43 dataInList = []
44
44
45 # objeto de datos de entrada (Voltage, Spectra o Correlation)
45 # objeto de datos de entrada (Voltage, Spectra o Correlation)
46 dataOut = None
46 dataOut = None
47
47
48 operations2RunDict = None
48 operations2RunDict = None
49
49
50 isConfig = False
50 isConfig = False
51
51
52
52
53 def __init__(self, *args, **kwargs):
53 def __init__(self, *args, **kwargs):
54
54
55 self.dataIn = None
55 self.dataIn = None
56 self.dataInList = []
56 self.dataInList = []
57
57
58 self.dataOut = None
58 self.dataOut = None
59
59
60 self.operations2RunDict = {}
60 self.operations2RunDict = {}
61 self.operationKwargs = {}
61 self.operationKwargs = {}
62
62
63 self.isConfig = False
63 self.isConfig = False
64
64
65 self.args = args
65 self.args = args
66 self.kwargs = kwargs
66 self.kwargs = kwargs
67 checkKwargs(self.run, kwargs)
67 checkKwargs(self.run, kwargs)
68
68
69 def getAllowedArgs(self):
69 def getAllowedArgs(self):
70 return inspect.getargspec(self.run).args
70 return inspect.getargspec(self.run).args
71
71
72 def addOperationKwargs(self, objId, **kwargs):
72 def addOperationKwargs(self, objId, **kwargs):
73 '''
73 '''
74 '''
74 '''
75
75
76 self.operationKwargs[objId] = kwargs
76 self.operationKwargs[objId] = kwargs
77
77
78
78
79 def addOperation(self, opObj, objId):
79 def addOperation(self, opObj, objId):
80
80
81 """
81 """
82 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
82 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
83 identificador asociado a este objeto.
83 identificador asociado a este objeto.
84
84
85 Input:
85 Input:
86
86
87 object : objeto de la clase "Operation"
87 object : objeto de la clase "Operation"
88
88
89 Return:
89 Return:
90
90
91 objId : identificador del objeto, necesario para ejecutar la operacion
91 objId : identificador del objeto, necesario para ejecutar la operacion
92 """
92 """
93
93
94 self.operations2RunDict[objId] = opObj
94 self.operations2RunDict[objId] = opObj
95
95
96 return objId
96 return objId
97
97
98 def getOperationObj(self, objId):
98 def getOperationObj(self, objId):
99
99
100 if objId not in self.operations2RunDict.keys():
100 if objId not in self.operations2RunDict.keys():
101 return None
101 return None
102
102
103 return self.operations2RunDict[objId]
103 return self.operations2RunDict[objId]
104
104
105 def operation(self, **kwargs):
105 def operation(self, **kwargs):
106
106
107 """
107 """
108 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
108 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
109 atributos del objeto dataOut
109 atributos del objeto dataOut
110
110
111 Input:
111 Input:
112
112
113 **kwargs : Diccionario de argumentos de la funcion a ejecutar
113 **kwargs : Diccionario de argumentos de la funcion a ejecutar
114 """
114 """
115
115
116 raise NotImplementedError
116 raise NotImplementedError
117
117
118 def callMethod(self, name, opId):
118 def callMethod(self, name, opId):
119
119
120 """
120 """
121 Ejecuta el metodo con el nombre "name" y con argumentos **kwargs de la propia clase.
121 Ejecuta el metodo con el nombre "name" y con argumentos **kwargs de la propia clase.
122
122
123 Input:
123 Input:
124 name : nombre del metodo a ejecutar
124 name : nombre del metodo a ejecutar
125
125
126 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
126 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
127
127
128 """
128 """
129
129
130 #Checking the inputs
130 #Checking the inputs
131 if name == 'run':
131 if name == 'run':
132
132
133 if not self.checkInputs():
133 if not self.checkInputs():
134 self.dataOut.flagNoData = True
134 self.dataOut.flagNoData = True
135 return False
135 return False
136 else:
136 else:
137 #Si no es un metodo RUN la entrada es la misma dataOut (interna)
137 #Si no es un metodo RUN la entrada es la misma dataOut (interna)
138 if self.dataOut is not None and self.dataOut.isEmpty():
138 if self.dataOut is not None and self.dataOut.isEmpty():
139 return False
139 return False
140
140
141 #Getting the pointer to method
141 #Getting the pointer to method
142 methodToCall = getattr(self, name)
142 methodToCall = getattr(self, name)
143
143
144 #Executing the self method
144 #Executing the self method
145
145
146 if hasattr(self, 'mp'):
146 if hasattr(self, 'mp'):
147 if name=='run':
147 if name=='run':
148 if self.mp is False:
148 if self.mp is False:
149 self.mp = True
149 self.mp = True
150 self.start()
150 self.start()
151 else:
151 else:
152 self.operationKwargs[opId]['parent'] = self.kwargs
152 self.operationKwargs[opId]['parent'] = self.kwargs
153 methodToCall(**self.operationKwargs[opId])
153 methodToCall(**self.operationKwargs[opId])
154 else:
154 else:
155 if name=='run':
155 if name=='run':
156 methodToCall(**self.kwargs)
156 methodToCall(**self.kwargs)
157 else:
157 else:
158 methodToCall(**self.operationKwargs[opId])
158 methodToCall(**self.operationKwargs[opId])
159
159
160 if self.dataOut is None:
160 if self.dataOut is None:
161 return False
161 return False
162
162
163 if self.dataOut.isEmpty():
163 if self.dataOut.isEmpty():
164 return False
164 return False
165
165
166 return True
166 return True
167
167
168 def callObject(self, objId):
168 def callObject(self, objId):
169
169
170 """
170 """
171 Ejecuta la operacion asociada al identificador del objeto "objId"
171 Ejecuta la operacion asociada al identificador del objeto "objId"
172
172
173 Input:
173 Input:
174
174
175 objId : identificador del objeto a ejecutar
175 objId : identificador del objeto a ejecutar
176
176
177 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
177 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
178
178
179 Return:
179 Return:
180
180
181 None
181 None
182 """
182 """
183
183
184 if self.dataOut is not None and self.dataOut.isEmpty():
184 if self.dataOut is not None and self.dataOut.isEmpty():
185 return False
185 return False
186
186
187 externalProcObj = self.operations2RunDict[objId]
187 externalProcObj = self.operations2RunDict[objId]
188
188
189 if hasattr(externalProcObj, 'mp'):
189 if hasattr(externalProcObj, 'mp'):
190 if externalProcObj.mp is False:
190 if externalProcObj.mp is False:
191 externalProcObj.kwargs['parent'] = self.kwargs
191 externalProcObj.kwargs['parent'] = self.kwargs
192 self.operationKwargs[objId] = externalProcObj.kwargs
192 self.operationKwargs[objId] = externalProcObj.kwargs
193 externalProcObj.mp = True
193 externalProcObj.mp = True
194 externalProcObj.start()
194 externalProcObj.start()
195 else:
195 else:
196 externalProcObj.run(self.dataOut, **externalProcObj.kwargs)
196 externalProcObj.run(self.dataOut, **externalProcObj.kwargs)
197 self.operationKwargs[objId] = externalProcObj.kwargs
197 self.operationKwargs[objId] = externalProcObj.kwargs
198
198
199
199
200 return True
200 return True
201
201
202 def call(self, opType, opName=None, opId=None):
202 def call(self, opType, opName=None, opId=None):
203
204 """
203 """
205 Return True si ejecuta la operacion interna nombrada "opName" o la operacion externa
204 Return True si ejecuta la operacion interna nombrada "opName" o la operacion externa
206 identificada con el id "opId"; con los argumentos "**kwargs".
205 identificada con el id "opId"; con los argumentos "**kwargs".
207
206
208 False si la operacion no se ha ejecutado.
207 False si la operacion no se ha ejecutado.
209
208
210 Input:
209 Input:
211
210
212 opType : Puede ser "self" o "external"
211 opType : Puede ser "self" o "external"
213
212
214 Depende del tipo de operacion para llamar a:callMethod or callObject:
213 Depende del tipo de operacion para llamar a:callMethod or callObject:
215
214
216 1. If opType = "self": Llama a un metodo propio de esta clase:
215 1. If opType = "self": Llama a un metodo propio de esta clase:
217
216
218 name_method = getattr(self, name)
217 name_method = getattr(self, name)
219 name_method(**kwargs)
218 name_method(**kwargs)
220
219
221
220
222 2. If opType = "other" o"external": Llama al metodo "run()" de una instancia de la
221 2. If opType = "other" o"external": Llama al metodo "run()" de una instancia de la
223 clase "Operation" o de un derivado de ella:
222 clase "Operation" o de un derivado de ella:
224
223
225 instanceName = self.operationList[opId]
224 instanceName = self.operationList[opId]
226 instanceName.run(**kwargs)
225 instanceName.run(**kwargs)
227
226
228 opName : Si la operacion es interna (opType = 'self'), entonces el "opName" sera
227 opName : Si la operacion es interna (opType = 'self'), entonces el "opName" sera
229 usada para llamar a un metodo interno de la clase Processing
228 usada para llamar a un metodo interno de la clase Processing
230
229
231 opId : Si la operacion es externa (opType = 'other' o 'external), entonces el
230 opId : Si la operacion es externa (opType = 'other' o 'external), entonces el
232 "opId" sera usada para llamar al metodo "run" de la clase Operation
231 "opId" sera usada para llamar al metodo "run" de la clase Operation
233 registrada anteriormente con ese Id
232 registrada anteriormente con ese Id
234
233
235 Exception:
234 Exception:
236 Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
235 Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
237 "addOperation" e identificado con el valor "opId" = el id de la operacion.
236 "addOperation" e identificado con el valor "opId" = el id de la operacion.
238 De lo contrario retornara un error del tipo ValueError
237 De lo contrario retornara un error del tipo ValueError
239
238
240 """
239 """
241
240
242 if opType == 'self':
241 if opType == 'self':
243
242
244 if not opName:
243 if not opName:
245 raise ValueError, "opName parameter should be defined"
244 raise ValueError, "opName parameter should be defined"
246
245
247 sts = self.callMethod(opName, opId)
246 sts = self.callMethod(opName, opId)
248
247
249 elif opType == 'other' or opType == 'external' or opType == 'plotter':
248 elif opType == 'other' or opType == 'external' or opType == 'plotter':
250
249
251 if not opId:
250 if not opId:
252 raise ValueError, "opId parameter should be defined"
251 raise ValueError, "opId parameter should be defined"
253
252
254 if opId not in self.operations2RunDict.keys():
253 if opId not in self.operations2RunDict.keys():
255 raise ValueError, "Any operation with id=%s has been added" %str(opId)
254 raise ValueError, "Any operation with id=%s has been added" %str(opId)
256
255
257 sts = self.callObject(opId)
256 sts = self.callObject(opId)
258
257
259 else:
258 else:
260 raise ValueError, "opType should be 'self', 'external' or 'plotter'; and not '%s'" %opType
259 raise ValueError, "opType should be 'self', 'external' or 'plotter'; and not '%s'" %opType
261
260
262 return sts
261 return sts
263
262
264 def setInput(self, dataIn):
263 def setInput(self, dataIn):
265
264
266 self.dataIn = dataIn
265 self.dataIn = dataIn
267 self.dataInList.append(dataIn)
266 self.dataInList.append(dataIn)
268
267
269 def getOutputObj(self):
268 def getOutputObj(self):
270
269
271 return self.dataOut
270 return self.dataOut
272
271
273 def checkInputs(self):
272 def checkInputs(self):
274
273
275 for thisDataIn in self.dataInList:
274 for thisDataIn in self.dataInList:
276
275
277 if thisDataIn.isEmpty():
276 if thisDataIn.isEmpty():
278 return False
277 return False
279
278
280 return True
279 return True
281
280
282 def setup(self):
281 def setup(self):
283
282
284 raise NotImplementedError
283 raise NotImplementedError
285
284
286 def run(self):
285 def run(self):
287
286
288 raise NotImplementedError
287 raise NotImplementedError
289
288
290 def close(self):
289 def close(self):
291 #Close every thread, queue or any other object here is it is neccesary.
290 #Close every thread, queue or any other object here is it is neccesary.
292 return
291 return
293
292
294 class Operation(object):
293 class Operation(object):
295
294
296 """
295 """
297 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
296 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
298 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
297 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
299 acumulacion dentro de esta clase
298 acumulacion dentro de esta clase
300
299
301 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
300 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
302
301
303 """
302 """
304
303
305 __buffer = None
304 __buffer = None
306 isConfig = False
305 isConfig = False
307
306
308 def __init__(self, **kwargs):
307 def __init__(self, **kwargs):
309
308
310 self.__buffer = None
309 self.__buffer = None
311 self.isConfig = False
310 self.isConfig = False
312 self.kwargs = kwargs
311 self.kwargs = kwargs
313 checkKwargs(self.run, kwargs)
312 checkKwargs(self.run, kwargs)
314
313
315 def getAllowedArgs(self):
314 def getAllowedArgs(self):
316 return inspect.getargspec(self.run).args
315 return inspect.getargspec(self.run).args
317
316
318 def setup(self):
317 def setup(self):
319
318
320 self.isConfig = True
319 self.isConfig = True
321
320
322 raise NotImplementedError
321 raise NotImplementedError
323
322
324 def run(self, dataIn, **kwargs):
323 def run(self, dataIn, **kwargs):
325
324
326 """
325 """
327 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
326 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
328 atributos del objeto dataIn.
327 atributos del objeto dataIn.
329
328
330 Input:
329 Input:
331
330
332 dataIn : objeto del tipo JROData
331 dataIn : objeto del tipo JROData
333
332
334 Return:
333 Return:
335
334
336 None
335 None
337
336
338 Affected:
337 Affected:
339 __buffer : buffer de recepcion de datos.
338 __buffer : buffer de recepcion de datos.
340
339
341 """
340 """
342 if not self.isConfig:
341 if not self.isConfig:
343 self.setup(**kwargs)
342 self.setup(**kwargs)
344
343
345 raise NotImplementedError
344 raise NotImplementedError
346
345
347 def close(self):
346 def close(self):
348
347
349 pass
348 pass
Show file before | Show file after | Hide comments
@@ -1,64 +1,64
1 """.
1 """.
2
2
3 Created on Jul 16, 2014
3 Created on Jul 16, 2014
4
4
5 @author: Miguel Urco
5 @author: Miguel Urco
6 """
6 """
7
7
8 import numpy
8 import numpy
9 from setuptools import setup, Extension
9 from setuptools import setup, Extension
10 import numpy
10 from schainpy import __version__
11
11
12 setup(name="schainpy",
12 setup(name="schainpy",
13 version=__version__,
13 version=__version__,
14 description="Python tools to read, write and process Jicamarca data",
14 description="Python tools to read, write and process Jicamarca data",
15 author="Miguel Urco",
15 author="Miguel Urco",
16 author_email="miguel.urco@jro.igp.gob.pe",
16 author_email="miguel.urco@jro.igp.gob.pe",
17 url="http://jro.igp.gob.pe",
17 url="http://jro.igp.gob.pe",
18 packages={'schainpy',
18 packages={'schainpy',
19 'schainpy.model',
19 'schainpy.model',
20 'schainpy.model.data',
20 'schainpy.model.data',
21 'schainpy.model.graphics',
21 'schainpy.model.graphics',
22 'schainpy.model.io',
22 'schainpy.model.io',
23 'schainpy.model.proc',
23 'schainpy.model.proc',
24 'schainpy.model.serializer',
24 'schainpy.model.serializer',
25 'schainpy.model.utils',
25 'schainpy.model.utils',
26 'schainpy.gui',
26 'schainpy.gui',
27 'schainpy.gui.figures',
27 'schainpy.gui.figures',
28 'schainpy.gui.viewcontroller',
28 'schainpy.gui.viewcontroller',
29 'schainpy.gui.viewer',
29 'schainpy.gui.viewer',
30 'schainpy.gui.viewer.windows'},
30 'schainpy.gui.viewer.windows'},
31 ext_package='schainpy',
31 ext_package='schainpy',
32 py_modules=[''],
32 py_modules=[''],
33 package_data={'': ['schain.conf.template'],
33 package_data={'': ['schain.conf.template'],
34 'schainpy.gui.figures': ['*.png', '*.jpg'],
34 'schainpy.gui.figures': ['*.png', '*.jpg'],
35 },
35 },
36 include_package_data=False,
36 include_package_data=False,
37 entry_points={
37 entry_points={
38 'console_scripts': [
38 'console_scripts': [
39 'schain = schaincli.cli:main',
39 'schain = schaincli.cli:main',
40 ],
40 ],
41 },
41 },
42 scripts=['schainpy/gui/schainGUI'],
42 scripts=['schainpy/gui/schainGUI'],
43 ext_modules=[
43 ext_modules=[
44 Extension("cSchain",
44 Extension("cSchain",
45 ["schainpy/model/proc/extensions.c"],
45 ["schainpy/model/proc/extensions.c"],
46 include_dirs=[numpy.get_include()],
46 include_dirs=[numpy.get_include()],
47 #extra_compile_args=['-Werror'],
47 #extra_compile_args=['-Werror'],
48 )
48 )
49 ],
49 ],
50 install_requires=[
50 install_requires=[
51 "scipy >= 0.14.0",
51 "scipy >= 0.14.0",
52 "h5py >= 2.2.1",
52 "h5py >= 2.2.1",
53 "matplotlib >= 1.4.2",
53 "matplotlib >= 1.4.2",
54 "pyfits >= 3.4",
54 "pyfits >= 3.4",
55 "paramiko >= 2.1.2",
55 "paramiko >= 2.1.2",
56 "paho-mqtt >= 1.2",
56 "paho-mqtt >= 1.2",
57 "zmq",
57 "zmq",
58 "fuzzywuzzy",
58 "fuzzywuzzy",
59 "click",
59 "click",
60 "colorama",
60 "colorama",
61 "python-Levenshtein"
61 "python-Levenshtein"
62 ],
62 ],
63 )
63 )
64
64
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now