##// END OF EJS Templates
schain-jc
RSS
Improve logger
jespinoza -
r1134:d5610a11112f
parent child
Show More
Side by Side Unified
Context file:
r1134:d5610a11112f Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1323 +1,1323
1 '''
1 '''
2 Created on September , 2012
2 Created on September , 2012
3 @author:
3 @author:
4 '''
4 '''
5
5
6 import sys
6 import sys
7 import ast
7 import ast
8 import datetime
8 import datetime
9 import traceback
9 import traceback
10 import math
10 import math
11 import time
11 import time
12 from multiprocessing import Process, cpu_count
12 from multiprocessing import Process, cpu_count
13
13
14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
15 from xml.dom import minidom
15 from xml.dom import minidom
16
16
17 import schainpy
17 import schainpy
18 import schainpy.admin
18 import schainpy.admin
19 from schainpy.model import *
19 from schainpy.model import *
20 from schainpy.utils import log
20 from schainpy.utils import log
21
21
22 DTYPES = {
22 DTYPES = {
23 'Voltage': '.r',
23 'Voltage': '.r',
24 'Spectra': '.pdata'
24 'Spectra': '.pdata'
25 }
25 }
26
26
27
27
28 def MPProject(project, n=cpu_count()):
28 def MPProject(project, n=cpu_count()):
29 '''
29 '''
30 Project wrapper to run schain in n processes
30 Project wrapper to run schain in n processes
31 '''
31 '''
32
32
33 rconf = project.getReadUnitObj()
33 rconf = project.getReadUnitObj()
34 op = rconf.getOperationObj('run')
34 op = rconf.getOperationObj('run')
35 dt1 = op.getParameterValue('startDate')
35 dt1 = op.getParameterValue('startDate')
36 dt2 = op.getParameterValue('endDate')
36 dt2 = op.getParameterValue('endDate')
37 tm1 = op.getParameterValue('startTime')
37 tm1 = op.getParameterValue('startTime')
38 tm2 = op.getParameterValue('endTime')
38 tm2 = op.getParameterValue('endTime')
39 days = (dt2 - dt1).days
39 days = (dt2 - dt1).days
40
40
41 for day in range(days + 1):
41 for day in range(days + 1):
42 skip = 0
42 skip = 0
43 cursor = 0
43 cursor = 0
44 processes = []
44 processes = []
45 dt = dt1 + datetime.timedelta(day)
45 dt = dt1 + datetime.timedelta(day)
46 dt_str = dt.strftime('%Y/%m/%d')
46 dt_str = dt.strftime('%Y/%m/%d')
47 reader = JRODataReader()
47 reader = JRODataReader()
48 paths, files = reader.searchFilesOffLine(path=rconf.path,
48 paths, files = reader.searchFilesOffLine(path=rconf.path,
49 startDate=dt,
49 startDate=dt,
50 endDate=dt,
50 endDate=dt,
51 startTime=tm1,
51 startTime=tm1,
52 endTime=tm2,
52 endTime=tm2,
53 ext=DTYPES[rconf.datatype])
53 ext=DTYPES[rconf.datatype])
54 nFiles = len(files)
54 nFiles = len(files)
55 if nFiles == 0:
55 if nFiles == 0:
56 continue
56 continue
57 skip = int(math.ceil(nFiles / n))
57 skip = int(math.ceil(nFiles / n))
58 while nFiles > cursor * skip:
58 while nFiles > cursor * skip:
59 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
59 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
60 skip=skip)
60 skip=skip)
61 p = project.clone()
61 p = project.clone()
62 p.start()
62 p.start()
63 processes.append(p)
63 processes.append(p)
64 cursor += 1
64 cursor += 1
65
65
66 def beforeExit(exctype, value, trace):
66 def beforeExit(exctype, value, trace):
67 for process in processes:
67 for process in processes:
68 process.terminate()
68 process.terminate()
69 process.join()
69 process.join()
70 print traceback.print_tb(trace)
70 print traceback.print_tb(trace)
71
71
72 sys.excepthook = beforeExit
72 sys.excepthook = beforeExit
73
73
74 for process in processes:
74 for process in processes:
75 process.join()
75 process.join()
76 process.terminate()
76 process.terminate()
77
77
78 time.sleep(3)
78 time.sleep(3)
79
79
80
80
81 class ParameterConf():
81 class ParameterConf():
82
82
83 id = None
83 id = None
84 name = None
84 name = None
85 value = None
85 value = None
86 format = None
86 format = None
87
87
88 __formated_value = None
88 __formated_value = None
89
89
90 ELEMENTNAME = 'Parameter'
90 ELEMENTNAME = 'Parameter'
91
91
92 def __init__(self):
92 def __init__(self):
93
93
94 self.format = 'str'
94 self.format = 'str'
95
95
96 def getElementName(self):
96 def getElementName(self):
97
97
98 return self.ELEMENTNAME
98 return self.ELEMENTNAME
99
99
100 def getValue(self):
100 def getValue(self):
101
101
102 value = self.value
102 value = self.value
103 format = self.format
103 format = self.format
104
104
105 if self.__formated_value != None:
105 if self.__formated_value != None:
106
106
107 return self.__formated_value
107 return self.__formated_value
108
108
109 if format == 'obj':
109 if format == 'obj':
110 return value
110 return value
111
111
112 if format == 'str':
112 if format == 'str':
113 self.__formated_value = str(value)
113 self.__formated_value = str(value)
114 return self.__formated_value
114 return self.__formated_value
115
115
116 if value == '':
116 if value == '':
117 raise ValueError, '%s: This parameter value is empty' % self.name
117 raise ValueError, '%s: This parameter value is empty' % self.name
118
118
119 if format == 'list':
119 if format == 'list':
120 strList = value.split(',')
120 strList = value.split(',')
121
121
122 self.__formated_value = strList
122 self.__formated_value = strList
123
123
124 return self.__formated_value
124 return self.__formated_value
125
125
126 if format == 'intlist':
126 if format == 'intlist':
127 '''
127 '''
128 Example:
128 Example:
129 value = (0,1,2)
129 value = (0,1,2)
130 '''
130 '''
131
131
132 new_value = ast.literal_eval(value)
132 new_value = ast.literal_eval(value)
133
133
134 if type(new_value) not in (tuple, list):
134 if type(new_value) not in (tuple, list):
135 new_value = [int(new_value)]
135 new_value = [int(new_value)]
136
136
137 self.__formated_value = new_value
137 self.__formated_value = new_value
138
138
139 return self.__formated_value
139 return self.__formated_value
140
140
141 if format == 'floatlist':
141 if format == 'floatlist':
142 '''
142 '''
143 Example:
143 Example:
144 value = (0.5, 1.4, 2.7)
144 value = (0.5, 1.4, 2.7)
145 '''
145 '''
146
146
147 new_value = ast.literal_eval(value)
147 new_value = ast.literal_eval(value)
148
148
149 if type(new_value) not in (tuple, list):
149 if type(new_value) not in (tuple, list):
150 new_value = [float(new_value)]
150 new_value = [float(new_value)]
151
151
152 self.__formated_value = new_value
152 self.__formated_value = new_value
153
153
154 return self.__formated_value
154 return self.__formated_value
155
155
156 if format == 'date':
156 if format == 'date':
157 strList = value.split('/')
157 strList = value.split('/')
158 intList = [int(x) for x in strList]
158 intList = [int(x) for x in strList]
159 date = datetime.date(intList[0], intList[1], intList[2])
159 date = datetime.date(intList[0], intList[1], intList[2])
160
160
161 self.__formated_value = date
161 self.__formated_value = date
162
162
163 return self.__formated_value
163 return self.__formated_value
164
164
165 if format == 'time':
165 if format == 'time':
166 strList = value.split(':')
166 strList = value.split(':')
167 intList = [int(x) for x in strList]
167 intList = [int(x) for x in strList]
168 time = datetime.time(intList[0], intList[1], intList[2])
168 time = datetime.time(intList[0], intList[1], intList[2])
169
169
170 self.__formated_value = time
170 self.__formated_value = time
171
171
172 return self.__formated_value
172 return self.__formated_value
173
173
174 if format == 'pairslist':
174 if format == 'pairslist':
175 '''
175 '''
176 Example:
176 Example:
177 value = (0,1),(1,2)
177 value = (0,1),(1,2)
178 '''
178 '''
179
179
180 new_value = ast.literal_eval(value)
180 new_value = ast.literal_eval(value)
181
181
182 if type(new_value) not in (tuple, list):
182 if type(new_value) not in (tuple, list):
183 raise ValueError, '%s has to be a tuple or list of pairs' % value
183 raise ValueError, '%s has to be a tuple or list of pairs' % value
184
184
185 if type(new_value[0]) not in (tuple, list):
185 if type(new_value[0]) not in (tuple, list):
186 if len(new_value) != 2:
186 if len(new_value) != 2:
187 raise ValueError, '%s has to be a tuple or list of pairs' % value
187 raise ValueError, '%s has to be a tuple or list of pairs' % value
188 new_value = [new_value]
188 new_value = [new_value]
189
189
190 for thisPair in new_value:
190 for thisPair in new_value:
191 if len(thisPair) != 2:
191 if len(thisPair) != 2:
192 raise ValueError, '%s has to be a tuple or list of pairs' % value
192 raise ValueError, '%s has to be a tuple or list of pairs' % value
193
193
194 self.__formated_value = new_value
194 self.__formated_value = new_value
195
195
196 return self.__formated_value
196 return self.__formated_value
197
197
198 if format == 'multilist':
198 if format == 'multilist':
199 '''
199 '''
200 Example:
200 Example:
201 value = (0,1,2),(3,4,5)
201 value = (0,1,2),(3,4,5)
202 '''
202 '''
203 multiList = ast.literal_eval(value)
203 multiList = ast.literal_eval(value)
204
204
205 if type(multiList[0]) == int:
205 if type(multiList[0]) == int:
206 multiList = ast.literal_eval('(' + value + ')')
206 multiList = ast.literal_eval('(' + value + ')')
207
207
208 self.__formated_value = multiList
208 self.__formated_value = multiList
209
209
210 return self.__formated_value
210 return self.__formated_value
211
211
212 if format == 'bool':
212 if format == 'bool':
213 value = int(value)
213 value = int(value)
214
214
215 if format == 'int':
215 if format == 'int':
216 value = float(value)
216 value = float(value)
217
217
218 format_func = eval(format)
218 format_func = eval(format)
219
219
220 self.__formated_value = format_func(value)
220 self.__formated_value = format_func(value)
221
221
222 return self.__formated_value
222 return self.__formated_value
223
223
224 def updateId(self, new_id):
224 def updateId(self, new_id):
225
225
226 self.id = str(new_id)
226 self.id = str(new_id)
227
227
228 def setup(self, id, name, value, format='str'):
228 def setup(self, id, name, value, format='str'):
229 self.id = str(id)
229 self.id = str(id)
230 self.name = name
230 self.name = name
231 if format == 'obj':
231 if format == 'obj':
232 self.value = value
232 self.value = value
233 else:
233 else:
234 self.value = str(value)
234 self.value = str(value)
235 self.format = str.lower(format)
235 self.format = str.lower(format)
236
236
237 self.getValue()
237 self.getValue()
238
238
239 return 1
239 return 1
240
240
241 def update(self, name, value, format='str'):
241 def update(self, name, value, format='str'):
242
242
243 self.name = name
243 self.name = name
244 self.value = str(value)
244 self.value = str(value)
245 self.format = format
245 self.format = format
246
246
247 def makeXml(self, opElement):
247 def makeXml(self, opElement):
248 if self.name not in ('queue',):
248 if self.name not in ('queue',):
249 parmElement = SubElement(opElement, self.ELEMENTNAME)
249 parmElement = SubElement(opElement, self.ELEMENTNAME)
250 parmElement.set('id', str(self.id))
250 parmElement.set('id', str(self.id))
251 parmElement.set('name', self.name)
251 parmElement.set('name', self.name)
252 parmElement.set('value', self.value)
252 parmElement.set('value', self.value)
253 parmElement.set('format', self.format)
253 parmElement.set('format', self.format)
254
254
255 def readXml(self, parmElement):
255 def readXml(self, parmElement):
256
256
257 self.id = parmElement.get('id')
257 self.id = parmElement.get('id')
258 self.name = parmElement.get('name')
258 self.name = parmElement.get('name')
259 self.value = parmElement.get('value')
259 self.value = parmElement.get('value')
260 self.format = str.lower(parmElement.get('format'))
260 self.format = str.lower(parmElement.get('format'))
261
261
262 # Compatible with old signal chain version
262 # Compatible with old signal chain version
263 if self.format == 'int' and self.name == 'idfigure':
263 if self.format == 'int' and self.name == 'idfigure':
264 self.name = 'id'
264 self.name = 'id'
265
265
266 def printattr(self):
266 def printattr(self):
267
267
268 print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
268 print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
269
269
270
270
271 class OperationConf():
271 class OperationConf():
272
272
273 id = None
273 id = None
274 name = None
274 name = None
275 priority = None
275 priority = None
276 type = None
276 type = None
277
277
278 parmConfObjList = []
278 parmConfObjList = []
279
279
280 ELEMENTNAME = 'Operation'
280 ELEMENTNAME = 'Operation'
281
281
282 def __init__(self):
282 def __init__(self):
283
283
284 self.id = '0'
284 self.id = '0'
285 self.name = None
285 self.name = None
286 self.priority = None
286 self.priority = None
287 self.type = 'self'
287 self.type = 'self'
288
288
289 def __getNewId(self):
289 def __getNewId(self):
290
290
291 return int(self.id) * 10 + len(self.parmConfObjList) + 1
291 return int(self.id) * 10 + len(self.parmConfObjList) + 1
292
292
293 def updateId(self, new_id):
293 def updateId(self, new_id):
294
294
295 self.id = str(new_id)
295 self.id = str(new_id)
296
296
297 n = 1
297 n = 1
298 for parmObj in self.parmConfObjList:
298 for parmObj in self.parmConfObjList:
299
299
300 idParm = str(int(new_id) * 10 + n)
300 idParm = str(int(new_id) * 10 + n)
301 parmObj.updateId(idParm)
301 parmObj.updateId(idParm)
302
302
303 n += 1
303 n += 1
304
304
305 def getElementName(self):
305 def getElementName(self):
306
306
307 return self.ELEMENTNAME
307 return self.ELEMENTNAME
308
308
309 def getParameterObjList(self):
309 def getParameterObjList(self):
310
310
311 return self.parmConfObjList
311 return self.parmConfObjList
312
312
313 def getParameterObj(self, parameterName):
313 def getParameterObj(self, parameterName):
314
314
315 for parmConfObj in self.parmConfObjList:
315 for parmConfObj in self.parmConfObjList:
316
316
317 if parmConfObj.name != parameterName:
317 if parmConfObj.name != parameterName:
318 continue
318 continue
319
319
320 return parmConfObj
320 return parmConfObj
321
321
322 return None
322 return None
323
323
324 def getParameterObjfromValue(self, parameterValue):
324 def getParameterObjfromValue(self, parameterValue):
325
325
326 for parmConfObj in self.parmConfObjList:
326 for parmConfObj in self.parmConfObjList:
327
327
328 if parmConfObj.getValue() != parameterValue:
328 if parmConfObj.getValue() != parameterValue:
329 continue
329 continue
330
330
331 return parmConfObj.getValue()
331 return parmConfObj.getValue()
332
332
333 return None
333 return None
334
334
335 def getParameterValue(self, parameterName):
335 def getParameterValue(self, parameterName):
336
336
337 parameterObj = self.getParameterObj(parameterName)
337 parameterObj = self.getParameterObj(parameterName)
338
338
339 # if not parameterObj:
339 # if not parameterObj:
340 # return None
340 # return None
341
341
342 value = parameterObj.getValue()
342 value = parameterObj.getValue()
343
343
344 return value
344 return value
345
345
346 def getKwargs(self):
346 def getKwargs(self):
347
347
348 kwargs = {}
348 kwargs = {}
349
349
350 for parmConfObj in self.parmConfObjList:
350 for parmConfObj in self.parmConfObjList:
351 if self.name == 'run' and parmConfObj.name == 'datatype':
351 if self.name == 'run' and parmConfObj.name == 'datatype':
352 continue
352 continue
353
353
354 kwargs[parmConfObj.name] = parmConfObj.getValue()
354 kwargs[parmConfObj.name] = parmConfObj.getValue()
355
355
356 return kwargs
356 return kwargs
357
357
358 def setup(self, id, name, priority, type):
358 def setup(self, id, name, priority, type):
359
359
360 self.id = str(id)
360 self.id = str(id)
361 self.name = name
361 self.name = name
362 self.type = type
362 self.type = type
363 self.priority = priority
363 self.priority = priority
364
364
365 self.parmConfObjList = []
365 self.parmConfObjList = []
366
366
367 def removeParameters(self):
367 def removeParameters(self):
368
368
369 for obj in self.parmConfObjList:
369 for obj in self.parmConfObjList:
370 del obj
370 del obj
371
371
372 self.parmConfObjList = []
372 self.parmConfObjList = []
373
373
374 def addParameter(self, name, value, format='str'):
374 def addParameter(self, name, value, format='str'):
375
375
376 if value is None:
376 if value is None:
377 return None
377 return None
378 id = self.__getNewId()
378 id = self.__getNewId()
379
379
380 parmConfObj = ParameterConf()
380 parmConfObj = ParameterConf()
381 if not parmConfObj.setup(id, name, value, format):
381 if not parmConfObj.setup(id, name, value, format):
382 return None
382 return None
383
383
384 self.parmConfObjList.append(parmConfObj)
384 self.parmConfObjList.append(parmConfObj)
385
385
386 return parmConfObj
386 return parmConfObj
387
387
388 def changeParameter(self, name, value, format='str'):
388 def changeParameter(self, name, value, format='str'):
389
389
390 parmConfObj = self.getParameterObj(name)
390 parmConfObj = self.getParameterObj(name)
391 parmConfObj.update(name, value, format)
391 parmConfObj.update(name, value, format)
392
392
393 return parmConfObj
393 return parmConfObj
394
394
395 def makeXml(self, procUnitElement):
395 def makeXml(self, procUnitElement):
396
396
397 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
397 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
398 opElement.set('id', str(self.id))
398 opElement.set('id', str(self.id))
399 opElement.set('name', self.name)
399 opElement.set('name', self.name)
400 opElement.set('type', self.type)
400 opElement.set('type', self.type)
401 opElement.set('priority', str(self.priority))
401 opElement.set('priority', str(self.priority))
402
402
403 for parmConfObj in self.parmConfObjList:
403 for parmConfObj in self.parmConfObjList:
404 parmConfObj.makeXml(opElement)
404 parmConfObj.makeXml(opElement)
405
405
406 def readXml(self, opElement):
406 def readXml(self, opElement):
407
407
408 self.id = opElement.get('id')
408 self.id = opElement.get('id')
409 self.name = opElement.get('name')
409 self.name = opElement.get('name')
410 self.type = opElement.get('type')
410 self.type = opElement.get('type')
411 self.priority = opElement.get('priority')
411 self.priority = opElement.get('priority')
412
412
413 # Compatible with old signal chain version
413 # Compatible with old signal chain version
414 # Use of 'run' method instead 'init'
414 # Use of 'run' method instead 'init'
415 if self.type == 'self' and self.name == 'init':
415 if self.type == 'self' and self.name == 'init':
416 self.name = 'run'
416 self.name = 'run'
417
417
418 self.parmConfObjList = []
418 self.parmConfObjList = []
419
419
420 parmElementList = opElement.iter(ParameterConf().getElementName())
420 parmElementList = opElement.iter(ParameterConf().getElementName())
421
421
422 for parmElement in parmElementList:
422 for parmElement in parmElementList:
423 parmConfObj = ParameterConf()
423 parmConfObj = ParameterConf()
424 parmConfObj.readXml(parmElement)
424 parmConfObj.readXml(parmElement)
425
425
426 # Compatible with old signal chain version
426 # Compatible with old signal chain version
427 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
427 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
428 if self.type != 'self' and self.name == 'Plot':
428 if self.type != 'self' and self.name == 'Plot':
429 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
429 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
430 self.name = parmConfObj.value
430 self.name = parmConfObj.value
431 continue
431 continue
432
432
433 self.parmConfObjList.append(parmConfObj)
433 self.parmConfObjList.append(parmConfObj)
434
434
435 def printattr(self):
435 def printattr(self):
436
436
437 print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
437 print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
438 self.id,
438 self.id,
439 self.name,
439 self.name,
440 self.type,
440 self.type,
441 self.priority)
441 self.priority)
442
442
443 for parmConfObj in self.parmConfObjList:
443 for parmConfObj in self.parmConfObjList:
444 parmConfObj.printattr()
444 parmConfObj.printattr()
445
445
446 def createObject(self, plotter_queue=None):
446 def createObject(self, plotter_queue=None):
447
447
448 if self.type == 'self':
448 if self.type == 'self':
449 raise ValueError, 'This operation type cannot be created'
449 raise ValueError, 'This operation type cannot be created'
450
450
451 if self.type == 'plotter':
451 if self.type == 'plotter':
452 if not plotter_queue:
452 if not plotter_queue:
453 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
453 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
454
454
455 opObj = Plotter(self.name, plotter_queue)
455 opObj = Plotter(self.name, plotter_queue)
456
456
457 if self.type == 'external' or self.type == 'other':
457 if self.type == 'external' or self.type == 'other':
458
458
459 className = eval(self.name)
459 className = eval(self.name)
460 kwargs = self.getKwargs()
460 kwargs = self.getKwargs()
461
461
462 opObj = className(**kwargs)
462 opObj = className(**kwargs)
463
463
464 return opObj
464 return opObj
465
465
466
466
467 class ProcUnitConf():
467 class ProcUnitConf():
468
468
469 id = None
469 id = None
470 name = None
470 name = None
471 datatype = None
471 datatype = None
472 inputId = None
472 inputId = None
473 parentId = None
473 parentId = None
474
474
475 opConfObjList = []
475 opConfObjList = []
476
476
477 procUnitObj = None
477 procUnitObj = None
478 opObjList = []
478 opObjList = []
479
479
480 ELEMENTNAME = 'ProcUnit'
480 ELEMENTNAME = 'ProcUnit'
481
481
482 def __init__(self):
482 def __init__(self):
483
483
484 self.id = None
484 self.id = None
485 self.datatype = None
485 self.datatype = None
486 self.name = None
486 self.name = None
487 self.inputId = None
487 self.inputId = None
488
488
489 self.opConfObjList = []
489 self.opConfObjList = []
490
490
491 self.procUnitObj = None
491 self.procUnitObj = None
492 self.opObjDict = {}
492 self.opObjDict = {}
493
493
494 def __getPriority(self):
494 def __getPriority(self):
495
495
496 return len(self.opConfObjList) + 1
496 return len(self.opConfObjList) + 1
497
497
498 def __getNewId(self):
498 def __getNewId(self):
499
499
500 return int(self.id) * 10 + len(self.opConfObjList) + 1
500 return int(self.id) * 10 + len(self.opConfObjList) + 1
501
501
502 def getElementName(self):
502 def getElementName(self):
503
503
504 return self.ELEMENTNAME
504 return self.ELEMENTNAME
505
505
506 def getId(self):
506 def getId(self):
507
507
508 return self.id
508 return self.id
509
509
510 def updateId(self, new_id, parentId=parentId):
510 def updateId(self, new_id, parentId=parentId):
511
511
512 new_id = int(parentId) * 10 + (int(self.id) % 10)
512 new_id = int(parentId) * 10 + (int(self.id) % 10)
513 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
513 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
514
514
515 # If this proc unit has not inputs
515 # If this proc unit has not inputs
516 if self.inputId == '0':
516 if self.inputId == '0':
517 new_inputId = 0
517 new_inputId = 0
518
518
519 n = 1
519 n = 1
520 for opConfObj in self.opConfObjList:
520 for opConfObj in self.opConfObjList:
521
521
522 idOp = str(int(new_id) * 10 + n)
522 idOp = str(int(new_id) * 10 + n)
523 opConfObj.updateId(idOp)
523 opConfObj.updateId(idOp)
524
524
525 n += 1
525 n += 1
526
526
527 self.parentId = str(parentId)
527 self.parentId = str(parentId)
528 self.id = str(new_id)
528 self.id = str(new_id)
529 self.inputId = str(new_inputId)
529 self.inputId = str(new_inputId)
530
530
531 def getInputId(self):
531 def getInputId(self):
532
532
533 return self.inputId
533 return self.inputId
534
534
535 def getOperationObjList(self):
535 def getOperationObjList(self):
536
536
537 return self.opConfObjList
537 return self.opConfObjList
538
538
539 def getOperationObj(self, name=None):
539 def getOperationObj(self, name=None):
540
540
541 for opConfObj in self.opConfObjList:
541 for opConfObj in self.opConfObjList:
542
542
543 if opConfObj.name != name:
543 if opConfObj.name != name:
544 continue
544 continue
545
545
546 return opConfObj
546 return opConfObj
547
547
548 return None
548 return None
549
549
550 def getOpObjfromParamValue(self, value=None):
550 def getOpObjfromParamValue(self, value=None):
551
551
552 for opConfObj in self.opConfObjList:
552 for opConfObj in self.opConfObjList:
553 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
553 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
554 continue
554 continue
555 return opConfObj
555 return opConfObj
556 return None
556 return None
557
557
558 def getProcUnitObj(self):
558 def getProcUnitObj(self):
559
559
560 return self.procUnitObj
560 return self.procUnitObj
561
561
562 def setup(self, id, name, datatype, inputId, parentId=None):
562 def setup(self, id, name, datatype, inputId, parentId=None):
563
563
564 # Compatible with old signal chain version
564 # Compatible with old signal chain version
565 if datatype == None and name == None:
565 if datatype == None and name == None:
566 raise ValueError, 'datatype or name should be defined'
566 raise ValueError, 'datatype or name should be defined'
567
567
568 if name == None:
568 if name == None:
569 if 'Proc' in datatype:
569 if 'Proc' in datatype:
570 name = datatype
570 name = datatype
571 else:
571 else:
572 name = '%sProc' % (datatype)
572 name = '%sProc' % (datatype)
573
573
574 if datatype == None:
574 if datatype == None:
575 datatype = name.replace('Proc', '')
575 datatype = name.replace('Proc', '')
576
576
577 self.id = str(id)
577 self.id = str(id)
578 self.name = name
578 self.name = name
579 self.datatype = datatype
579 self.datatype = datatype
580 self.inputId = inputId
580 self.inputId = inputId
581 self.parentId = parentId
581 self.parentId = parentId
582
582
583 self.opConfObjList = []
583 self.opConfObjList = []
584
584
585 self.addOperation(name='run', optype='self')
585 self.addOperation(name='run', optype='self')
586
586
587 def removeOperations(self):
587 def removeOperations(self):
588
588
589 for obj in self.opConfObjList:
589 for obj in self.opConfObjList:
590 del obj
590 del obj
591
591
592 self.opConfObjList = []
592 self.opConfObjList = []
593 self.addOperation(name='run')
593 self.addOperation(name='run')
594
594
595 def addParameter(self, **kwargs):
595 def addParameter(self, **kwargs):
596 '''
596 '''
597 Add parameters to 'run' operation
597 Add parameters to 'run' operation
598 '''
598 '''
599 opObj = self.opConfObjList[0]
599 opObj = self.opConfObjList[0]
600
600
601 opObj.addParameter(**kwargs)
601 opObj.addParameter(**kwargs)
602
602
603 return opObj
603 return opObj
604
604
605 def addOperation(self, name, optype='self'):
605 def addOperation(self, name, optype='self'):
606
606
607 id = self.__getNewId()
607 id = self.__getNewId()
608 priority = self.__getPriority()
608 priority = self.__getPriority()
609
609
610 opConfObj = OperationConf()
610 opConfObj = OperationConf()
611 opConfObj.setup(id, name=name, priority=priority, type=optype)
611 opConfObj.setup(id, name=name, priority=priority, type=optype)
612
612
613 self.opConfObjList.append(opConfObj)
613 self.opConfObjList.append(opConfObj)
614
614
615 return opConfObj
615 return opConfObj
616
616
617 def makeXml(self, projectElement):
617 def makeXml(self, projectElement):
618
618
619 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
619 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
620 procUnitElement.set('id', str(self.id))
620 procUnitElement.set('id', str(self.id))
621 procUnitElement.set('name', self.name)
621 procUnitElement.set('name', self.name)
622 procUnitElement.set('datatype', self.datatype)
622 procUnitElement.set('datatype', self.datatype)
623 procUnitElement.set('inputId', str(self.inputId))
623 procUnitElement.set('inputId', str(self.inputId))
624
624
625 for opConfObj in self.opConfObjList:
625 for opConfObj in self.opConfObjList:
626 opConfObj.makeXml(procUnitElement)
626 opConfObj.makeXml(procUnitElement)
627
627
628 def readXml(self, upElement):
628 def readXml(self, upElement):
629
629
630 self.id = upElement.get('id')
630 self.id = upElement.get('id')
631 self.name = upElement.get('name')
631 self.name = upElement.get('name')
632 self.datatype = upElement.get('datatype')
632 self.datatype = upElement.get('datatype')
633 self.inputId = upElement.get('inputId')
633 self.inputId = upElement.get('inputId')
634
634
635 if self.ELEMENTNAME == 'ReadUnit':
635 if self.ELEMENTNAME == 'ReadUnit':
636 self.datatype = self.datatype.replace('Reader', '')
636 self.datatype = self.datatype.replace('Reader', '')
637
637
638 if self.ELEMENTNAME == 'ProcUnit':
638 if self.ELEMENTNAME == 'ProcUnit':
639 self.datatype = self.datatype.replace('Proc', '')
639 self.datatype = self.datatype.replace('Proc', '')
640
640
641 if self.inputId == 'None':
641 if self.inputId == 'None':
642 self.inputId = '0'
642 self.inputId = '0'
643
643
644 self.opConfObjList = []
644 self.opConfObjList = []
645
645
646 opElementList = upElement.iter(OperationConf().getElementName())
646 opElementList = upElement.iter(OperationConf().getElementName())
647
647
648 for opElement in opElementList:
648 for opElement in opElementList:
649 opConfObj = OperationConf()
649 opConfObj = OperationConf()
650 opConfObj.readXml(opElement)
650 opConfObj.readXml(opElement)
651 self.opConfObjList.append(opConfObj)
651 self.opConfObjList.append(opConfObj)
652
652
653 def printattr(self):
653 def printattr(self):
654
654
655 print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
655 print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
656 self.id,
656 self.id,
657 self.name,
657 self.name,
658 self.datatype,
658 self.datatype,
659 self.inputId)
659 self.inputId)
660
660
661 for opConfObj in self.opConfObjList:
661 for opConfObj in self.opConfObjList:
662 opConfObj.printattr()
662 opConfObj.printattr()
663
663
664 def getKwargs(self):
664 def getKwargs(self):
665
665
666 opObj = self.opConfObjList[0]
666 opObj = self.opConfObjList[0]
667 kwargs = opObj.getKwargs()
667 kwargs = opObj.getKwargs()
668
668
669 return kwargs
669 return kwargs
670
670
671 def createObjects(self, plotter_queue=None):
671 def createObjects(self, plotter_queue=None):
672
672
673 className = eval(self.name)
673 className = eval(self.name)
674 kwargs = self.getKwargs()
674 kwargs = self.getKwargs()
675 procUnitObj = className(**kwargs)
675 procUnitObj = className(**kwargs)
676
676
677 for opConfObj in self.opConfObjList:
677 for opConfObj in self.opConfObjList:
678
678
679 if opConfObj.type == 'self' and self.name == 'run':
679 if opConfObj.type == 'self' and self.name == 'run':
680 continue
680 continue
681 elif opConfObj.type == 'self':
681 elif opConfObj.type == 'self':
682 procUnitObj.addOperationKwargs(
682 procUnitObj.addOperationKwargs(
683 opConfObj.id, **opConfObj.getKwargs())
683 opConfObj.id, **opConfObj.getKwargs())
684 continue
684 continue
685
685
686 opObj = opConfObj.createObject(plotter_queue)
686 opObj = opConfObj.createObject(plotter_queue)
687
687
688 self.opObjDict[opConfObj.id] = opObj
688 self.opObjDict[opConfObj.id] = opObj
689
689
690 procUnitObj.addOperation(opObj, opConfObj.id)
690 procUnitObj.addOperation(opObj, opConfObj.id)
691
691
692 self.procUnitObj = procUnitObj
692 self.procUnitObj = procUnitObj
693
693
694 return procUnitObj
694 return procUnitObj
695
695
696 def run(self):
696 def run(self):
697
697
698 is_ok = False
698 is_ok = False
699
699
700 for opConfObj in self.opConfObjList:
700 for opConfObj in self.opConfObjList:
701
701
702 kwargs = {}
702 kwargs = {}
703 for parmConfObj in opConfObj.getParameterObjList():
703 for parmConfObj in opConfObj.getParameterObjList():
704 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
704 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
705 continue
705 continue
706
706
707 kwargs[parmConfObj.name] = parmConfObj.getValue()
707 kwargs[parmConfObj.name] = parmConfObj.getValue()
708
708
709 sts = self.procUnitObj.call(opType=opConfObj.type,
709 sts = self.procUnitObj.call(opType=opConfObj.type,
710 opName=opConfObj.name,
710 opName=opConfObj.name,
711 opId=opConfObj.id)
711 opId=opConfObj.id)
712
712
713 is_ok = is_ok or sts
713 is_ok = is_ok or sts
714
714
715 return is_ok
715 return is_ok
716
716
717 def close(self):
717 def close(self):
718
718
719 for opConfObj in self.opConfObjList:
719 for opConfObj in self.opConfObjList:
720 if opConfObj.type == 'self':
720 if opConfObj.type == 'self':
721 continue
721 continue
722
722
723 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
723 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
724 opObj.close()
724 opObj.close()
725
725
726 self.procUnitObj.close()
726 self.procUnitObj.close()
727
727
728 return
728 return
729
729
730
730
731 class ReadUnitConf(ProcUnitConf):
731 class ReadUnitConf(ProcUnitConf):
732
732
733 path = None
733 path = None
734 startDate = None
734 startDate = None
735 endDate = None
735 endDate = None
736 startTime = None
736 startTime = None
737 endTime = None
737 endTime = None
738
738
739 ELEMENTNAME = 'ReadUnit'
739 ELEMENTNAME = 'ReadUnit'
740
740
741 def __init__(self):
741 def __init__(self):
742
742
743 self.id = None
743 self.id = None
744 self.datatype = None
744 self.datatype = None
745 self.name = None
745 self.name = None
746 self.inputId = None
746 self.inputId = None
747
747
748 self.parentId = None
748 self.parentId = None
749
749
750 self.opConfObjList = []
750 self.opConfObjList = []
751 self.opObjList = []
751 self.opObjList = []
752
752
753 def getElementName(self):
753 def getElementName(self):
754
754
755 return self.ELEMENTNAME
755 return self.ELEMENTNAME
756
756
757 def setup(self, id, name, datatype, path='', startDate='', endDate='',
757 def setup(self, id, name, datatype, path='', startDate='', endDate='',
758 startTime='', endTime='', parentId=None, server=None, **kwargs):
758 startTime='', endTime='', parentId=None, server=None, **kwargs):
759
759
760 # Compatible with old signal chain version
760 # Compatible with old signal chain version
761 if datatype == None and name == None:
761 if datatype == None and name == None:
762 raise ValueError, 'datatype or name should be defined'
762 raise ValueError, 'datatype or name should be defined'
763 if name == None:
763 if name == None:
764 if 'Reader' in datatype:
764 if 'Reader' in datatype:
765 name = datatype
765 name = datatype
766 datatype = name.replace('Reader','')
766 datatype = name.replace('Reader','')
767 else:
767 else:
768 name = '{}Reader'.format(datatype)
768 name = '{}Reader'.format(datatype)
769 if datatype == None:
769 if datatype == None:
770 if 'Reader' in name:
770 if 'Reader' in name:
771 datatype = name.replace('Reader','')
771 datatype = name.replace('Reader','')
772 else:
772 else:
773 datatype = name
773 datatype = name
774 name = '{}Reader'.format(name)
774 name = '{}Reader'.format(name)
775
775
776 self.id = id
776 self.id = id
777 self.name = name
777 self.name = name
778 self.datatype = datatype
778 self.datatype = datatype
779 if path != '':
779 if path != '':
780 self.path = os.path.abspath(path)
780 self.path = os.path.abspath(path)
781 self.startDate = startDate
781 self.startDate = startDate
782 self.endDate = endDate
782 self.endDate = endDate
783 self.startTime = startTime
783 self.startTime = startTime
784 self.endTime = endTime
784 self.endTime = endTime
785 self.inputId = '0'
785 self.inputId = '0'
786 self.parentId = parentId
786 self.parentId = parentId
787 self.server = server
787 self.server = server
788 self.addRunOperation(**kwargs)
788 self.addRunOperation(**kwargs)
789
789
790 def update(self, **kwargs):
790 def update(self, **kwargs):
791
791
792 if 'datatype' in kwargs:
792 if 'datatype' in kwargs:
793 datatype = kwargs.pop('datatype')
793 datatype = kwargs.pop('datatype')
794 if 'Reader' in datatype:
794 if 'Reader' in datatype:
795 self.name = datatype
795 self.name = datatype
796 else:
796 else:
797 self.name = '%sReader' % (datatype)
797 self.name = '%sReader' % (datatype)
798 self.datatype = self.name.replace('Reader', '')
798 self.datatype = self.name.replace('Reader', '')
799
799
800 attrs = ('path', 'startDate', 'endDate',
800 attrs = ('path', 'startDate', 'endDate',
801 'startTime', 'endTime', 'parentId')
801 'startTime', 'endTime', 'parentId')
802
802
803 for attr in attrs:
803 for attr in attrs:
804 if attr in kwargs:
804 if attr in kwargs:
805 setattr(self, attr, kwargs.pop(attr))
805 setattr(self, attr, kwargs.pop(attr))
806
806
807 self.inputId = '0'
807 self.inputId = '0'
808 self.updateRunOperation(**kwargs)
808 self.updateRunOperation(**kwargs)
809
809
810 def removeOperations(self):
810 def removeOperations(self):
811
811
812 for obj in self.opConfObjList:
812 for obj in self.opConfObjList:
813 del obj
813 del obj
814
814
815 self.opConfObjList = []
815 self.opConfObjList = []
816
816
817 def addRunOperation(self, **kwargs):
817 def addRunOperation(self, **kwargs):
818
818
819 opObj = self.addOperation(name='run', optype='self')
819 opObj = self.addOperation(name='run', optype='self')
820
820
821 if self.server is None:
821 if self.server is None:
822 opObj.addParameter(
822 opObj.addParameter(
823 name='datatype', value=self.datatype, format='str')
823 name='datatype', value=self.datatype, format='str')
824 opObj.addParameter(name='path', value=self.path, format='str')
824 opObj.addParameter(name='path', value=self.path, format='str')
825 opObj.addParameter(
825 opObj.addParameter(
826 name='startDate', value=self.startDate, format='date')
826 name='startDate', value=self.startDate, format='date')
827 opObj.addParameter(
827 opObj.addParameter(
828 name='endDate', value=self.endDate, format='date')
828 name='endDate', value=self.endDate, format='date')
829 opObj.addParameter(
829 opObj.addParameter(
830 name='startTime', value=self.startTime, format='time')
830 name='startTime', value=self.startTime, format='time')
831 opObj.addParameter(
831 opObj.addParameter(
832 name='endTime', value=self.endTime, format='time')
832 name='endTime', value=self.endTime, format='time')
833
833
834 for key, value in kwargs.items():
834 for key, value in kwargs.items():
835 opObj.addParameter(name=key, value=value,
835 opObj.addParameter(name=key, value=value,
836 format=type(value).__name__)
836 format=type(value).__name__)
837 else:
837 else:
838 opObj.addParameter(name='server', value=self.server, format='str')
838 opObj.addParameter(name='server', value=self.server, format='str')
839
839
840 return opObj
840 return opObj
841
841
842 def updateRunOperation(self, **kwargs):
842 def updateRunOperation(self, **kwargs):
843
843
844 opObj = self.getOperationObj(name='run')
844 opObj = self.getOperationObj(name='run')
845 opObj.removeParameters()
845 opObj.removeParameters()
846
846
847 opObj.addParameter(name='datatype', value=self.datatype, format='str')
847 opObj.addParameter(name='datatype', value=self.datatype, format='str')
848 opObj.addParameter(name='path', value=self.path, format='str')
848 opObj.addParameter(name='path', value=self.path, format='str')
849 opObj.addParameter(
849 opObj.addParameter(
850 name='startDate', value=self.startDate, format='date')
850 name='startDate', value=self.startDate, format='date')
851 opObj.addParameter(name='endDate', value=self.endDate, format='date')
851 opObj.addParameter(name='endDate', value=self.endDate, format='date')
852 opObj.addParameter(
852 opObj.addParameter(
853 name='startTime', value=self.startTime, format='time')
853 name='startTime', value=self.startTime, format='time')
854 opObj.addParameter(name='endTime', value=self.endTime, format='time')
854 opObj.addParameter(name='endTime', value=self.endTime, format='time')
855
855
856 for key, value in kwargs.items():
856 for key, value in kwargs.items():
857 opObj.addParameter(name=key, value=value,
857 opObj.addParameter(name=key, value=value,
858 format=type(value).__name__)
858 format=type(value).__name__)
859
859
860 return opObj
860 return opObj
861
861
862 def readXml(self, upElement):
862 def readXml(self, upElement):
863
863
864 self.id = upElement.get('id')
864 self.id = upElement.get('id')
865 self.name = upElement.get('name')
865 self.name = upElement.get('name')
866 self.datatype = upElement.get('datatype')
866 self.datatype = upElement.get('datatype')
867 self.inputId = upElement.get('inputId')
867 self.inputId = upElement.get('inputId')
868
868
869 if self.ELEMENTNAME == 'ReadUnit':
869 if self.ELEMENTNAME == 'ReadUnit':
870 self.datatype = self.datatype.replace('Reader', '')
870 self.datatype = self.datatype.replace('Reader', '')
871
871
872 if self.inputId == 'None':
872 if self.inputId == 'None':
873 self.inputId = '0'
873 self.inputId = '0'
874
874
875 self.opConfObjList = []
875 self.opConfObjList = []
876
876
877 opElementList = upElement.iter(OperationConf().getElementName())
877 opElementList = upElement.iter(OperationConf().getElementName())
878
878
879 for opElement in opElementList:
879 for opElement in opElementList:
880 opConfObj = OperationConf()
880 opConfObj = OperationConf()
881 opConfObj.readXml(opElement)
881 opConfObj.readXml(opElement)
882 self.opConfObjList.append(opConfObj)
882 self.opConfObjList.append(opConfObj)
883
883
884 if opConfObj.name == 'run':
884 if opConfObj.name == 'run':
885 self.path = opConfObj.getParameterValue('path')
885 self.path = opConfObj.getParameterValue('path')
886 self.startDate = opConfObj.getParameterValue('startDate')
886 self.startDate = opConfObj.getParameterValue('startDate')
887 self.endDate = opConfObj.getParameterValue('endDate')
887 self.endDate = opConfObj.getParameterValue('endDate')
888 self.startTime = opConfObj.getParameterValue('startTime')
888 self.startTime = opConfObj.getParameterValue('startTime')
889 self.endTime = opConfObj.getParameterValue('endTime')
889 self.endTime = opConfObj.getParameterValue('endTime')
890
890
891
891
892 class Project(Process):
892 class Project(Process):
893
893
894 id = None
894 id = None
895 # name = None
895 # name = None
896 description = None
896 description = None
897 filename = None
897 filename = None
898
898
899 procUnitConfObjDict = None
899 procUnitConfObjDict = None
900
900
901 ELEMENTNAME = 'Project'
901 ELEMENTNAME = 'Project'
902
902
903 plotterQueue = None
903 plotterQueue = None
904
904
905 def __init__(self, plotter_queue=None):
905 def __init__(self, plotter_queue=None):
906
906
907 Process.__init__(self)
907 Process.__init__(self)
908 self.id = None
908 self.id = None
909 # self.name = None
909 # self.name = None
910 self.description = None
910 self.description = None
911
911
912 self.plotterQueue = plotter_queue
912 self.plotterQueue = plotter_queue
913
913
914 self.procUnitConfObjDict = {}
914 self.procUnitConfObjDict = {}
915
915
916 def __getNewId(self):
916 def __getNewId(self):
917
917
918 idList = self.procUnitConfObjDict.keys()
918 idList = self.procUnitConfObjDict.keys()
919
919
920 id = int(self.id) * 10
920 id = int(self.id) * 10
921
921
922 while True:
922 while True:
923 id += 1
923 id += 1
924
924
925 if str(id) in idList:
925 if str(id) in idList:
926 continue
926 continue
927
927
928 break
928 break
929
929
930 return str(id)
930 return str(id)
931
931
932 def getElementName(self):
932 def getElementName(self):
933
933
934 return self.ELEMENTNAME
934 return self.ELEMENTNAME
935
935
936 def getId(self):
936 def getId(self):
937
937
938 return self.id
938 return self.id
939
939
940 def updateId(self, new_id):
940 def updateId(self, new_id):
941
941
942 self.id = str(new_id)
942 self.id = str(new_id)
943
943
944 keyList = self.procUnitConfObjDict.keys()
944 keyList = self.procUnitConfObjDict.keys()
945 keyList.sort()
945 keyList.sort()
946
946
947 n = 1
947 n = 1
948 newProcUnitConfObjDict = {}
948 newProcUnitConfObjDict = {}
949
949
950 for procKey in keyList:
950 for procKey in keyList:
951
951
952 procUnitConfObj = self.procUnitConfObjDict[procKey]
952 procUnitConfObj = self.procUnitConfObjDict[procKey]
953 idProcUnit = str(int(self.id) * 10 + n)
953 idProcUnit = str(int(self.id) * 10 + n)
954 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
954 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
955
955
956 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
956 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
957 n += 1
957 n += 1
958
958
959 self.procUnitConfObjDict = newProcUnitConfObjDict
959 self.procUnitConfObjDict = newProcUnitConfObjDict
960
960
961 def setup(self, id, name='', description=''):
961 def setup(self, id, name='', description=''):
962
962
963 print
963 print
964 print '*' * 60
964 print '*' * 60
965 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
965 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
966 print '*' * 60
966 print '*' * 60
967 print
967 print
968 self.id = str(id)
968 self.id = str(id)
969 self.description = description
969 self.description = description
970
970
971 def update(self, name, description):
971 def update(self, name, description):
972
972
973 self.description = description
973 self.description = description
974
974
975 def clone(self):
975 def clone(self):
976
976
977 p = Project()
977 p = Project()
978 p.procUnitConfObjDict = self.procUnitConfObjDict
978 p.procUnitConfObjDict = self.procUnitConfObjDict
979 return p
979 return p
980
980
981 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
981 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
982
982
983 if id is None:
983 if id is None:
984 idReadUnit = self.__getNewId()
984 idReadUnit = self.__getNewId()
985 else:
985 else:
986 idReadUnit = str(id)
986 idReadUnit = str(id)
987
987
988 readUnitConfObj = ReadUnitConf()
988 readUnitConfObj = ReadUnitConf()
989 readUnitConfObj.setup(idReadUnit, name, datatype,
989 readUnitConfObj.setup(idReadUnit, name, datatype,
990 parentId=self.id, **kwargs)
990 parentId=self.id, **kwargs)
991
991
992 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
992 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
993
993
994 return readUnitConfObj
994 return readUnitConfObj
995
995
996 def addProcUnit(self, inputId='0', datatype=None, name=None):
996 def addProcUnit(self, inputId='0', datatype=None, name=None):
997
997
998 idProcUnit = self.__getNewId()
998 idProcUnit = self.__getNewId()
999
999
1000 procUnitConfObj = ProcUnitConf()
1000 procUnitConfObj = ProcUnitConf()
1001 procUnitConfObj.setup(idProcUnit, name, datatype,
1001 procUnitConfObj.setup(idProcUnit, name, datatype,
1002 inputId, parentId=self.id)
1002 inputId, parentId=self.id)
1003
1003
1004 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1004 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1005
1005
1006 return procUnitConfObj
1006 return procUnitConfObj
1007
1007
1008 def removeProcUnit(self, id):
1008 def removeProcUnit(self, id):
1009
1009
1010 if id in self.procUnitConfObjDict.keys():
1010 if id in self.procUnitConfObjDict.keys():
1011 self.procUnitConfObjDict.pop(id)
1011 self.procUnitConfObjDict.pop(id)
1012
1012
1013 def getReadUnitId(self):
1013 def getReadUnitId(self):
1014
1014
1015 readUnitConfObj = self.getReadUnitObj()
1015 readUnitConfObj = self.getReadUnitObj()
1016
1016
1017 return readUnitConfObj.id
1017 return readUnitConfObj.id
1018
1018
1019 def getReadUnitObj(self):
1019 def getReadUnitObj(self):
1020
1020
1021 for obj in self.procUnitConfObjDict.values():
1021 for obj in self.procUnitConfObjDict.values():
1022 if obj.getElementName() == 'ReadUnit':
1022 if obj.getElementName() == 'ReadUnit':
1023 return obj
1023 return obj
1024
1024
1025 return None
1025 return None
1026
1026
1027 def getProcUnitObj(self, id=None, name=None):
1027 def getProcUnitObj(self, id=None, name=None):
1028
1028
1029 if id != None:
1029 if id != None:
1030 return self.procUnitConfObjDict[id]
1030 return self.procUnitConfObjDict[id]
1031
1031
1032 if name != None:
1032 if name != None:
1033 return self.getProcUnitObjByName(name)
1033 return self.getProcUnitObjByName(name)
1034
1034
1035 return None
1035 return None
1036
1036
1037 def getProcUnitObjByName(self, name):
1037 def getProcUnitObjByName(self, name):
1038
1038
1039 for obj in self.procUnitConfObjDict.values():
1039 for obj in self.procUnitConfObjDict.values():
1040 if obj.name == name:
1040 if obj.name == name:
1041 return obj
1041 return obj
1042
1042
1043 return None
1043 return None
1044
1044
1045 def procUnitItems(self):
1045 def procUnitItems(self):
1046
1046
1047 return self.procUnitConfObjDict.items()
1047 return self.procUnitConfObjDict.items()
1048
1048
1049 def makeXml(self):
1049 def makeXml(self):
1050
1050
1051 projectElement = Element('Project')
1051 projectElement = Element('Project')
1052 projectElement.set('id', str(self.id))
1052 projectElement.set('id', str(self.id))
1053 projectElement.set('name', self.name)
1053 projectElement.set('name', self.name)
1054 projectElement.set('description', self.description)
1054 projectElement.set('description', self.description)
1055
1055
1056 for procUnitConfObj in self.procUnitConfObjDict.values():
1056 for procUnitConfObj in self.procUnitConfObjDict.values():
1057 procUnitConfObj.makeXml(projectElement)
1057 procUnitConfObj.makeXml(projectElement)
1058
1058
1059 self.projectElement = projectElement
1059 self.projectElement = projectElement
1060
1060
1061 def writeXml(self, filename=None):
1061 def writeXml(self, filename=None):
1062
1062
1063 if filename == None:
1063 if filename == None:
1064 if self.filename:
1064 if self.filename:
1065 filename = self.filename
1065 filename = self.filename
1066 else:
1066 else:
1067 filename = 'schain.xml'
1067 filename = 'schain.xml'
1068
1068
1069 if not filename:
1069 if not filename:
1070 print 'filename has not been defined. Use setFilename(filename) for do it.'
1070 print 'filename has not been defined. Use setFilename(filename) for do it.'
1071 return 0
1071 return 0
1072
1072
1073 abs_file = os.path.abspath(filename)
1073 abs_file = os.path.abspath(filename)
1074
1074
1075 if not os.access(os.path.dirname(abs_file), os.W_OK):
1075 if not os.access(os.path.dirname(abs_file), os.W_OK):
1076 print 'No write permission on %s' % os.path.dirname(abs_file)
1076 print 'No write permission on %s' % os.path.dirname(abs_file)
1077 return 0
1077 return 0
1078
1078
1079 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1079 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1080 print 'File %s already exists and it could not be overwriten' % abs_file
1080 print 'File %s already exists and it could not be overwriten' % abs_file
1081 return 0
1081 return 0
1082
1082
1083 self.makeXml()
1083 self.makeXml()
1084
1084
1085 ElementTree(self.projectElement).write(abs_file, method='xml')
1085 ElementTree(self.projectElement).write(abs_file, method='xml')
1086
1086
1087 self.filename = abs_file
1087 self.filename = abs_file
1088
1088
1089 return 1
1089 return 1
1090
1090
1091 def readXml(self, filename=None):
1091 def readXml(self, filename=None):
1092
1092
1093 if not filename:
1093 if not filename:
1094 print 'filename is not defined'
1094 print 'filename is not defined'
1095 return 0
1095 return 0
1096
1096
1097 abs_file = os.path.abspath(filename)
1097 abs_file = os.path.abspath(filename)
1098
1098
1099 if not os.path.isfile(abs_file):
1099 if not os.path.isfile(abs_file):
1100 print '%s file does not exist' % abs_file
1100 print '%s file does not exist' % abs_file
1101 return 0
1101 return 0
1102
1102
1103 self.projectElement = None
1103 self.projectElement = None
1104 self.procUnitConfObjDict = {}
1104 self.procUnitConfObjDict = {}
1105
1105
1106 try:
1106 try:
1107 self.projectElement = ElementTree().parse(abs_file)
1107 self.projectElement = ElementTree().parse(abs_file)
1108 except:
1108 except:
1109 print 'Error reading %s, verify file format' % filename
1109 print 'Error reading %s, verify file format' % filename
1110 return 0
1110 return 0
1111
1111
1112 self.project = self.projectElement.tag
1112 self.project = self.projectElement.tag
1113
1113
1114 self.id = self.projectElement.get('id')
1114 self.id = self.projectElement.get('id')
1115 self.name = self.projectElement.get('name')
1115 self.name = self.projectElement.get('name')
1116 self.description = self.projectElement.get('description')
1116 self.description = self.projectElement.get('description')
1117
1117
1118 readUnitElementList = self.projectElement.iter(
1118 readUnitElementList = self.projectElement.iter(
1119 ReadUnitConf().getElementName())
1119 ReadUnitConf().getElementName())
1120
1120
1121 for readUnitElement in readUnitElementList:
1121 for readUnitElement in readUnitElementList:
1122 readUnitConfObj = ReadUnitConf()
1122 readUnitConfObj = ReadUnitConf()
1123 readUnitConfObj.readXml(readUnitElement)
1123 readUnitConfObj.readXml(readUnitElement)
1124
1124
1125 if readUnitConfObj.parentId == None:
1125 if readUnitConfObj.parentId == None:
1126 readUnitConfObj.parentId = self.id
1126 readUnitConfObj.parentId = self.id
1127
1127
1128 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1128 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1129
1129
1130 procUnitElementList = self.projectElement.iter(
1130 procUnitElementList = self.projectElement.iter(
1131 ProcUnitConf().getElementName())
1131 ProcUnitConf().getElementName())
1132
1132
1133 for procUnitElement in procUnitElementList:
1133 for procUnitElement in procUnitElementList:
1134 procUnitConfObj = ProcUnitConf()
1134 procUnitConfObj = ProcUnitConf()
1135 procUnitConfObj.readXml(procUnitElement)
1135 procUnitConfObj.readXml(procUnitElement)
1136
1136
1137 if procUnitConfObj.parentId == None:
1137 if procUnitConfObj.parentId == None:
1138 procUnitConfObj.parentId = self.id
1138 procUnitConfObj.parentId = self.id
1139
1139
1140 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1140 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1141
1141
1142 self.filename = abs_file
1142 self.filename = abs_file
1143
1143
1144 return 1
1144 return 1
1145
1145
1146 def printattr(self):
1146 def printattr(self):
1147
1147
1148 print 'Project[%s]: name = %s, description = %s' % (self.id,
1148 print 'Project[%s]: name = %s, description = %s' % (self.id,
1149 self.name,
1149 self.name,
1150 self.description)
1150 self.description)
1151
1151
1152 for procUnitConfObj in self.procUnitConfObjDict.values():
1152 for procUnitConfObj in self.procUnitConfObjDict.values():
1153 procUnitConfObj.printattr()
1153 procUnitConfObj.printattr()
1154
1154
1155 def createObjects(self):
1155 def createObjects(self):
1156
1156
1157 for procUnitConfObj in self.procUnitConfObjDict.values():
1157 for procUnitConfObj in self.procUnitConfObjDict.values():
1158 procUnitConfObj.createObjects(self.plotterQueue)
1158 procUnitConfObj.createObjects(self.plotterQueue)
1159
1159
1160 def __connect(self, objIN, thisObj):
1160 def __connect(self, objIN, thisObj):
1161
1161
1162 thisObj.setInput(objIN.getOutputObj())
1162 thisObj.setInput(objIN.getOutputObj())
1163
1163
1164 def connectObjects(self):
1164 def connectObjects(self):
1165
1165
1166 for thisPUConfObj in self.procUnitConfObjDict.values():
1166 for thisPUConfObj in self.procUnitConfObjDict.values():
1167
1167
1168 inputId = thisPUConfObj.getInputId()
1168 inputId = thisPUConfObj.getInputId()
1169
1169
1170 if int(inputId) == 0:
1170 if int(inputId) == 0:
1171 continue
1171 continue
1172
1172
1173 # Get input object
1173 # Get input object
1174 puConfINObj = self.procUnitConfObjDict[inputId]
1174 puConfINObj = self.procUnitConfObjDict[inputId]
1175 puObjIN = puConfINObj.getProcUnitObj()
1175 puObjIN = puConfINObj.getProcUnitObj()
1176
1176
1177 # Get current object
1177 # Get current object
1178 thisPUObj = thisPUConfObj.getProcUnitObj()
1178 thisPUObj = thisPUConfObj.getProcUnitObj()
1179
1179
1180 self.__connect(puObjIN, thisPUObj)
1180 self.__connect(puObjIN, thisPUObj)
1181
1181
1182 def __handleError(self, procUnitConfObj, send_email=False):
1182 def __handleError(self, procUnitConfObj, send_email=False):
1183
1183
1184 import socket
1184 import socket
1185
1185
1186 err = traceback.format_exception(sys.exc_info()[0],
1186 err = traceback.format_exception(sys.exc_info()[0],
1187 sys.exc_info()[1],
1187 sys.exc_info()[1],
1188 sys.exc_info()[2])
1188 sys.exc_info()[2])
1189
1189
1190 print '***** Error occurred in %s *****' % (procUnitConfObj.name)
1190 print '***** Error occurred in %s *****' % (procUnitConfObj.name)
1191 print '***** %s' % err[-1]
1191 print '***** %s' % err[-1]
1192
1192
1193 message = ''.join(err)
1193 message = ''.join(err)
1194
1194
1195 sys.stderr.write(message)
1195 sys.stderr.write(message)
1196
1196
1197 if not send_email:
1197 if not send_email:
1198 return
1198 return
1199
1199
1200 subject = 'SChain v%s: Error running %s\n' % (
1200 subject = 'SChain v%s: Error running %s\n' % (
1201 schainpy.__version__, procUnitConfObj.name)
1201 schainpy.__version__, procUnitConfObj.name)
1202
1202
1203 subtitle = '%s: %s\n' % (
1203 subtitle = '%s: %s\n' % (
1204 procUnitConfObj.getElementName(), procUnitConfObj.name)
1204 procUnitConfObj.getElementName(), procUnitConfObj.name)
1205 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1205 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1206 socket.gethostname())
1206 socket.gethostname())
1207 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1207 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1208 subtitle += 'Configuration file: %s\n' % self.filename
1208 subtitle += 'Configuration file: %s\n' % self.filename
1209 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1209 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1210
1210
1211 readUnitConfObj = self.getReadUnitObj()
1211 readUnitConfObj = self.getReadUnitObj()
1212 if readUnitConfObj:
1212 if readUnitConfObj:
1213 subtitle += '\nInput parameters:\n'
1213 subtitle += '\nInput parameters:\n'
1214 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1214 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1215 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1215 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1216 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1216 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1217 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1217 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1218 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1218 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1219 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1219 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1220
1220
1221 adminObj = schainpy.admin.SchainNotify()
1221 adminObj = schainpy.admin.SchainNotify()
1222 adminObj.sendAlert(message=message,
1222 adminObj.sendAlert(message=message,
1223 subject=subject,
1223 subject=subject,
1224 subtitle=subtitle,
1224 subtitle=subtitle,
1225 filename=self.filename)
1225 filename=self.filename)
1226
1226
1227 def isPaused(self):
1227 def isPaused(self):
1228 return 0
1228 return 0
1229
1229
1230 def isStopped(self):
1230 def isStopped(self):
1231 return 0
1231 return 0
1232
1232
1233 def runController(self):
1233 def runController(self):
1234 '''
1234 '''
1235 returns 0 when this process has been stopped, 1 otherwise
1235 returns 0 when this process has been stopped, 1 otherwise
1236 '''
1236 '''
1237
1237
1238 if self.isPaused():
1238 if self.isPaused():
1239 print 'Process suspended'
1239 print 'Process suspended'
1240
1240
1241 while True:
1241 while True:
1242 time.sleep(0.1)
1242 time.sleep(0.1)
1243
1243
1244 if not self.isPaused():
1244 if not self.isPaused():
1245 break
1245 break
1246
1246
1247 if self.isStopped():
1247 if self.isStopped():
1248 break
1248 break
1249
1249
1250 print 'Process reinitialized'
1250 print 'Process reinitialized'
1251
1251
1252 if self.isStopped():
1252 if self.isStopped():
1253 print 'Process stopped'
1253 print 'Process stopped'
1254 return 0
1254 return 0
1255
1255
1256 return 1
1256 return 1
1257
1257
1258 def setFilename(self, filename):
1258 def setFilename(self, filename):
1259
1259
1260 self.filename = filename
1260 self.filename = filename
1261
1261
1262 def setPlotterQueue(self, plotter_queue):
1262 def setPlotterQueue(self, plotter_queue):
1263
1263
1264 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1264 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1265
1265
1266 def getPlotterQueue(self):
1266 def getPlotterQueue(self):
1267
1267
1268 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1268 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1269
1269
1270 def useExternalPlotter(self):
1270 def useExternalPlotter(self):
1271
1271
1272 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1272 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1273
1273
1274 def run(self):
1274 def run(self):
1275
1275
1276 log.success('Starting {}'.format(self.name))
1276 log.success('Starting {}'.format(self.name), tag='')
1277 self.start_time = time.time()
1277 self.start_time = time.time()
1278 self.createObjects()
1278 self.createObjects()
1279 self.connectObjects()
1279 self.connectObjects()
1280
1280
1281 keyList = self.procUnitConfObjDict.keys()
1281 keyList = self.procUnitConfObjDict.keys()
1282 keyList.sort()
1282 keyList.sort()
1283
1283
1284 while(True):
1284 while(True):
1285
1285
1286 is_ok = False
1286 is_ok = False
1287
1287
1288 for procKey in keyList:
1288 for procKey in keyList:
1289
1289
1290 procUnitConfObj = self.procUnitConfObjDict[procKey]
1290 procUnitConfObj = self.procUnitConfObjDict[procKey]
1291
1291
1292 try:
1292 try:
1293 sts = procUnitConfObj.run()
1293 sts = procUnitConfObj.run()
1294 is_ok = is_ok or sts
1294 is_ok = is_ok or sts
1295 except KeyboardInterrupt:
1295 except KeyboardInterrupt:
1296 is_ok = False
1296 is_ok = False
1297 break
1297 break
1298 except ValueError, e:
1298 except ValueError, e:
1299 time.sleep(0.5)
1299 time.sleep(0.5)
1300 self.__handleError(procUnitConfObj, send_email=True)
1300 self.__handleError(procUnitConfObj, send_email=True)
1301 is_ok = False
1301 is_ok = False
1302 break
1302 break
1303 except:
1303 except:
1304 time.sleep(0.5)
1304 time.sleep(0.5)
1305 self.__handleError(procUnitConfObj)
1305 self.__handleError(procUnitConfObj)
1306 is_ok = False
1306 is_ok = False
1307 break
1307 break
1308
1308
1309 # If every process unit finished so end process
1309 # If every process unit finished so end process
1310 if not(is_ok):
1310 if not(is_ok):
1311 break
1311 break
1312
1312
1313 if not self.runController():
1313 if not self.runController():
1314 break
1314 break
1315
1315
1316 # Closing every process
1316 # Closing every process
1317 for procKey in keyList:
1317 for procKey in keyList:
1318 procUnitConfObj = self.procUnitConfObjDict[procKey]
1318 procUnitConfObj = self.procUnitConfObjDict[procKey]
1319 procUnitConfObj.close()
1319 procUnitConfObj.close()
1320
1320
1321 log.success('{} finished (time: {}s)'.format(
1321 log.success('{} finished (time: {}s)'.format(
1322 self.name,
1322 self.name,
1323 time.time()-self.start_time))
1323 time.time()-self.start_time))
Show file before | Show file after | Hide comments
@@ -1,45 +1,57
1 '''
1 '''
2 SCHAINPY - LOG
2 SCHAINPY - LOG
3 Simple helper for log standarization
3 Simple helper for log standarization
4 Usage:
4 Usage:
5 from schainpy.utils import log
5 from schainpy.utils import log
6 log.error('A kitten died beacuse of you')
6 log.error('A kitten died beacuse of you')
7 log.warning('You are doing it wrong but what the heck, I'll allow it)
7 log.warning('You are doing it wrong but what the heck, I'll allow it)
8 log.succes('YOU ROCK!')
8 log.succes('YOU ROCK!')
9 To create your own logger inside your class do it like this:
9 To create your own logger inside your class do it like this:
10 from schainpy.utils import log
10 from schainpy.utils import log
11 awesomeLogger = log.makelogger("never gonna", bg="red", fg="white")
11 awesomeLogger = log.makelogger("never gonna", bg="red", fg="white")
12 awesomeLogger('give you up')
12 awesomeLogger('give you up')
13 which will look like this:
13 which will look like this:
14 [NEVER GONNA] - give you up
14 [NEVER GONNA] - give you up
15 with color red as background and white as foreground.
15 with color red as background and white as foreground.
16 '''
16 '''
17
17
18 import click
18 import click
19
19
20
20
21 def warning(message, tag='Warning'):
21 def warning(message, tag='Warning', nl=True):
22 click.echo(click.style('[{}] {}'.format(tag, message), fg='yellow'))
22 if tag:
23 click.echo(click.style('[{}] {}'.format(tag, message), fg='yellow'), nl=nl)
24 else:
25 click.echo(click.style('{}'.format(message), fg='yellow'), nl=nl)
23 pass
26 pass
24
27
25
28
26 def error(message, tag='Error'):
29 def error(message, tag='Error', nl=True):
27 click.echo(click.style('[{}] {}'.format(tag, message), fg='red'))
30 if tag:
31 click.echo(click.style('[{}] {}'.format(tag, message), fg='red'), nl=nl)
32 else:
33 click.echo(click.style('{}'.format(message), fg='red'), nl=nl)
28 pass
34 pass
29
35
30
36
31 def success(message, tag='Info'):
37 def success(message, tag='Success', nl=True):
32 click.echo(click.style('[{}] {}'.format(tag, message), fg='green'))
38 if tag:
39 click.echo(click.style('[{}] {}'.format(tag, message), fg='green'), nl=nl)
40 else:
41 click.echo(click.style('{}'.format(message), fg='green'), nl=nl)
33 pass
42 pass
34
43
35
44
36 def log(message, tag='Info'):
45 def log(message, tag='Info', nl=True):
37 click.echo('[{}] {}'.format(tag, message))
46 if tag:
47 click.echo('[{}] {}'.format(tag, message), nl=nl)
48 else:
49 click.echo('{}'.format(message), nl=nl)
38 pass
50 pass
39
51
40
52
41 def makelogger(tag, bg='reset', fg='reset'):
53 def makelogger(tag, bg='reset', fg='reset'):
42 def func(message):
54 def func(message):
43 click.echo(click.style('[{}] {}'.format(
55 click.echo(click.style('[{}] {}'.format(
44 tag.upper(), message), bg=bg, fg=fg))
56 tag.upper(), message), bg=bg, fg=fg))
45 return func
57 return func
General Comments 0
You need to be logged in to leave comments. Login now