##// END OF EJS Templates
schain
RSS
merge from master
José Chávez -
r1014:c5cbd628ce0f merge
parent child
Show More
Side by Side Unified
Context file:
r1014:c5cbd628ce0f Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,69
1 {
2 "configurations": [
3 {
4 "name": "Mac",
5 "includePath": [
6 "/usr/include",
7 "/usr/local/include",
8 "${workspaceRoot}"
9 ],
10 "defines": [],
11 "intelliSenseMode": "clang-x64",
12 "browse": {
13 "path": [
14 "/usr/include",
15 "/usr/local/include",
16 "${workspaceRoot}"
17 ],
18 "limitSymbolsToIncludedHeaders": true,
19 "databaseFilename": ""
20 }
21 },
22 {
23 "name": "Linux",
24 "includePath": [
25 "/usr/include/c++/4.8.4",
26 "/usr/include/x86_64-linux-gnu/c++/4.8",
27 "/usr/local/include",
28 "/usr/include",
29 "/usr/include/x86_64-linux-gnu",
30 "${workspaceRoot}"
31 ],
32 "defines": [],
33 "intelliSenseMode": "clang-x64",
34 "browse": {
35 "path": [
36 "/usr/include/c++/4.8.4",
37 "/usr/include/x86_64-linux-gnu/c++/4.8",
38 "/usr/local/include",
39 "/usr/include",
40 "/usr/include/x86_64-linux-gnu",
41 "${workspaceRoot}"
42 ],
43 "limitSymbolsToIncludedHeaders": true,
44 "databaseFilename": ""
45 }
46 },
47 {
48 "name": "Win32",
49 "includePath": [
50 "C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/include",
51 "${workspaceRoot}"
52 ],
53 "defines": [
54 "_DEBUG",
55 "UNICODE"
56 ],
57 "intelliSenseMode": "msvc-x64",
58 "browse": {
59 "path": [
60 "C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/include/*",
61 "${workspaceRoot}"
62 ],
63 "limitSymbolsToIncludedHeaders": true,
64 "databaseFilename": ""
65 }
66 }
67 ],
68 "version": 2
69 } No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,7
1 {
2 "files.associations": {
3 "arrayobject.h": "c",
4 "ndarrayobject.h": "c",
5 "complex": "c"
6 }
7 } No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,109 +1,118
1 # Byte-compiled / optimized / DLL files
1 # Byte-compiled / optimized / DLL files
2 __pycache__/
2 __pycache__/
3 *.py[cod]
3 *.py[cod]
4 *$py.class
4 *$py.class
5
5
6 # C extensions
6 # C extensions
7 *.so
7 *.so
8
8
9 # Distribution / packaging
9 # Distribution / packaging
10 .Python
10 .Python
11 env/
11 env/
12 build/
12 build/
13 develop-eggs/
13 develop-eggs/
14 dist/
14 dist/
15 downloads/
15 downloads/
16 eggs/
16 eggs/
17 .eggs/
17 .eggs/
18 lib/
18 lib/
19 lib64/
19 lib64/
20 parts/
20 parts/
21 sdist/
21 sdist/
22 var/
22 var/
23 wheels/
23 wheels/
24 *.egg-info/
24 *.egg-info/
25 .installed.cfg
25 .installed.cfg
26 *.egg
26 *.egg
27
27
28 # PyInstaller
28 # PyInstaller
29 # Usually these files are written by a python script from a template
29 # Usually these files are written by a python script from a template
30 # before PyInstaller builds the exe, so as to inject date/other infos into it.
30 # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 *.manifest
31 *.manifest
32 *.spec
32 *.spec
33
33
34 # Installer logs
34 # Installer logs
35 pip-log.txt
35 pip-log.txt
36 pip-delete-this-directory.txt
36 pip-delete-this-directory.txt
37
37
38 # Unit test / coverage reports
38 # Unit test / coverage reports
39 htmlcov/
39 htmlcov/
40 .tox/
40 .tox/
41 .coverage
41 .coverage
42 .coverage.*
42 .coverage.*
43 .cache
43 .cache
44 nosetests.xml
44 nosetests.xml
45 coverage.xml
45 coverage.xml
46 *,cover
46 *,cover
47 .hypothesis/
47 .hypothesis/
48
48
49 # Translations
49 # Translations
50 *.mo
50 *.mo
51 *.pot
51 *.pot
52
52
53 # Django stuff:
53 # Django stuff:
54 *.log
54 *.log
55 local_settings.py
55 local_settings.py
56
56
57 # Flask stuff:
57 # Flask stuff:
58 instance/
58 instance/
59 .webassets-cache
59 .webassets-cache
60
60
61 # Scrapy stuff:
61 # Scrapy stuff:
62 .scrapy
62 .scrapy
63
63
64 # Sphinx documentation
64 # Sphinx documentation
65 docs/_build/
65 docs/_build/
66
66
67 # PyBuilder
67 # PyBuilder
68 target/
68 target/
69
69
70 # Jupyter Notebook
70 # Jupyter Notebook
71 .ipynb_checkpoints
71 .ipynb_checkpoints
72
72
73 # pyenv
73 # pyenv
74 .python-version
74 .python-version
75
75
76 # celery beat schedule file
76 # celery beat schedule file
77 celerybeat-schedule
77 celerybeat-schedule
78
78
79 # SageMath parsed files
79 # SageMath parsed files
80 *.sage.py
80 *.sage.py
81
81
82 # dotenv
82 # dotenv
83 .env
83 .env
84
84
85 # virtualenv
85 # virtualenv
86 .venv
86 .venv
87 venv/
87 venv/
88 ENV/
88 ENV/
89
89
90 # Spyder project settings
90 # Spyder project settings
91 .spyderproject
91 .spyderproject
92 .spyproject
92 .spyproject
93
93
94 # Rope project settings
94 # Rope project settings
95 .ropeproject
95 .ropeproject
96
96
97 # mkdocs documentation
97 # mkdocs documentation
98 /site
98 /site
99
99
100 # eclipse
100 # eclipse
101 .project
101 .project
102 .pydevproject
102 .pydevproject
103 <<<<<<< HEAD
103
104
104 # vscode
105 # vscode
105
106
106 .vscode
107 .vscode
107
108
108 schainpy/scripts/
109 schainpy/scripts/
109 schaingui/node_modules/
110 schaingui/node_modules/
111 =======
112 .svn/
113 *.png
114 *.pyc
115 schainpy/scripts
116
117 schaingui/node_modules
118 >>>>>>> master
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
Show file before | Show file after | Hide comments
@@ -1,1333 +1,1333
1 '''
1 '''
2 Created on September , 2012
2 Created on September , 2012
3 @author:
3 @author:
4 '''
4 '''
5
5
6 import sys
6 import sys
7 import ast
7 import ast
8 import datetime
8 import datetime
9 import traceback
9 import traceback
10 import math
10 import math
11 import time
11 import time
12 from multiprocessing import Process, Queue, cpu_count
12 from multiprocessing import Process, Queue, cpu_count
13 from profilehooks import profile, coverage
13 from profilehooks import profile, coverage
14
14
15 import schainpy
15 import schainpy
16 import schainpy.admin
16 import schainpy.admin
17
17
18 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
18 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
19 from xml.dom import minidom
19 from xml.dom import minidom
20
20
21 from schainpy.model import *
21 from schainpy.model import *
22 from time import sleep
22 from time import sleep
23
23
24
24
25
25
26 def prettify(elem):
26 def prettify(elem):
27 """Return a pretty-printed XML string for the Element.
27 """Return a pretty-printed XML string for the Element.
28 """
28 """
29 rough_string = tostring(elem, 'utf-8')
29 rough_string = tostring(elem, 'utf-8')
30 reparsed = minidom.parseString(rough_string)
30 reparsed = minidom.parseString(rough_string)
31 return reparsed.toprettyxml(indent=" ")
31 return reparsed.toprettyxml(indent=" ")
32
32
33 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
33 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
34 skip = 0
34 skip = 0
35 cursor = 0
35 cursor = 0
36 nFiles = None
36 nFiles = None
37 processes = []
37 processes = []
38 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
38 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
39 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
39 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
40 days = (dt2 - dt1).days
40 days = (dt2 - dt1).days
41
41
42 for day in range(days+1):
42 for day in range(days+1):
43 skip = 0
43 skip = 0
44 cursor = 0
44 cursor = 0
45 q = Queue()
45 q = Queue()
46 processes = []
46 processes = []
47 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
47 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
48 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
48 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
49 firstProcess.start()
49 firstProcess.start()
50 if by_day:
50 if by_day:
51 continue
51 continue
52 nFiles = q.get()
52 nFiles = q.get()
53 if nFiles==0:
53 if nFiles==0:
54 continue
54 continue
55 firstProcess.terminate()
55 firstProcess.terminate()
56 skip = int(math.ceil(nFiles/nProcess))
56 skip = int(math.ceil(nFiles/nProcess))
57 while True:
57 while True:
58 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
58 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
59 processes[cursor].start()
59 processes[cursor].start()
60 if nFiles < cursor*skip:
60 if nFiles < cursor*skip:
61 break
61 break
62 cursor += 1
62 cursor += 1
63
63
64 def beforeExit(exctype, value, trace):
64 def beforeExit(exctype, value, trace):
65 for process in processes:
65 for process in processes:
66 process.terminate()
66 process.terminate()
67 process.join()
67 process.join()
68 print traceback.print_tb(trace)
68 print traceback.print_tb(trace)
69
69
70 sys.excepthook = beforeExit
70 sys.excepthook = beforeExit
71
71
72 for process in processes:
72 for process in processes:
73 process.join()
73 process.join()
74 process.terminate()
74 process.terminate()
75
75
76 time.sleep(3)
76 time.sleep(3)
77
77
78
78
79 class ParameterConf():
79 class ParameterConf():
80
80
81 id = None
81 id = None
82 name = None
82 name = None
83 value = None
83 value = None
84 format = None
84 format = None
85
85
86 __formated_value = None
86 __formated_value = None
87
87
88 ELEMENTNAME = 'Parameter'
88 ELEMENTNAME = 'Parameter'
89
89
90 def __init__(self):
90 def __init__(self):
91
91
92 self.format = 'str'
92 self.format = 'str'
93
93
94 def getElementName(self):
94 def getElementName(self):
95
95
96 return self.ELEMENTNAME
96 return self.ELEMENTNAME
97
97
98 def getValue(self):
98 def getValue(self):
99
99
100 value = self.value
100 value = self.value
101 format = self.format
101 format = self.format
102
102
103 if self.__formated_value != None:
103 if self.__formated_value != None:
104
104
105 return self.__formated_value
105 return self.__formated_value
106
106
107 if format == 'obj':
107 if format == 'obj':
108 return value
108 return value
109
109
110 if format == 'str':
110 if format == 'str':
111 self.__formated_value = str(value)
111 self.__formated_value = str(value)
112 return self.__formated_value
112 return self.__formated_value
113
113
114 if value == '':
114 if value == '':
115 raise ValueError, "%s: This parameter value is empty" %self.name
115 raise ValueError, "%s: This parameter value is empty" %self.name
116
116
117 if format == 'list':
117 if format == 'list':
118 strList = value.split(',')
118 strList = value.split(',')
119
119
120 self.__formated_value = strList
120 self.__formated_value = strList
121
121
122 return self.__formated_value
122 return self.__formated_value
123
123
124 if format == 'intlist':
124 if format == 'intlist':
125 """
125 """
126 Example:
126 Example:
127 value = (0,1,2)
127 value = (0,1,2)
128 """
128 """
129
129
130 new_value = ast.literal_eval(value)
130 new_value = ast.literal_eval(value)
131
131
132 if type(new_value) not in (tuple, list):
132 if type(new_value) not in (tuple, list):
133 new_value = [int(new_value)]
133 new_value = [int(new_value)]
134
134
135 self.__formated_value = new_value
135 self.__formated_value = new_value
136
136
137 return self.__formated_value
137 return self.__formated_value
138
138
139 if format == 'floatlist':
139 if format == 'floatlist':
140 """
140 """
141 Example:
141 Example:
142 value = (0.5, 1.4, 2.7)
142 value = (0.5, 1.4, 2.7)
143 """
143 """
144
144
145 new_value = ast.literal_eval(value)
145 new_value = ast.literal_eval(value)
146
146
147 if type(new_value) not in (tuple, list):
147 if type(new_value) not in (tuple, list):
148 new_value = [float(new_value)]
148 new_value = [float(new_value)]
149
149
150 self.__formated_value = new_value
150 self.__formated_value = new_value
151
151
152 return self.__formated_value
152 return self.__formated_value
153
153
154 if format == 'date':
154 if format == 'date':
155 strList = value.split('/')
155 strList = value.split('/')
156 intList = [int(x) for x in strList]
156 intList = [int(x) for x in strList]
157 date = datetime.date(intList[0], intList[1], intList[2])
157 date = datetime.date(intList[0], intList[1], intList[2])
158
158
159 self.__formated_value = date
159 self.__formated_value = date
160
160
161 return self.__formated_value
161 return self.__formated_value
162
162
163 if format == 'time':
163 if format == 'time':
164 strList = value.split(':')
164 strList = value.split(':')
165 intList = [int(x) for x in strList]
165 intList = [int(x) for x in strList]
166 time = datetime.time(intList[0], intList[1], intList[2])
166 time = datetime.time(intList[0], intList[1], intList[2])
167
167
168 self.__formated_value = time
168 self.__formated_value = time
169
169
170 return self.__formated_value
170 return self.__formated_value
171
171
172 if format == 'pairslist':
172 if format == 'pairslist':
173 """
173 """
174 Example:
174 Example:
175 value = (0,1),(1,2)
175 value = (0,1),(1,2)
176 """
176 """
177
177
178 new_value = ast.literal_eval(value)
178 new_value = ast.literal_eval(value)
179
179
180 if type(new_value) not in (tuple, list):
180 if type(new_value) not in (tuple, list):
181 raise ValueError, "%s has to be a tuple or list of pairs" %value
181 raise ValueError, "%s has to be a tuple or list of pairs" %value
182
182
183 if type(new_value[0]) not in (tuple, list):
183 if type(new_value[0]) not in (tuple, list):
184 if len(new_value) != 2:
184 if len(new_value) != 2:
185 raise ValueError, "%s has to be a tuple or list of pairs" %value
185 raise ValueError, "%s has to be a tuple or list of pairs" %value
186 new_value = [new_value]
186 new_value = [new_value]
187
187
188 for thisPair in new_value:
188 for thisPair in new_value:
189 if len(thisPair) != 2:
189 if len(thisPair) != 2:
190 raise ValueError, "%s has to be a tuple or list of pairs" %value
190 raise ValueError, "%s has to be a tuple or list of pairs" %value
191
191
192 self.__formated_value = new_value
192 self.__formated_value = new_value
193
193
194 return self.__formated_value
194 return self.__formated_value
195
195
196 if format == 'multilist':
196 if format == 'multilist':
197 """
197 """
198 Example:
198 Example:
199 value = (0,1,2),(3,4,5)
199 value = (0,1,2),(3,4,5)
200 """
200 """
201 multiList = ast.literal_eval(value)
201 multiList = ast.literal_eval(value)
202
202
203 if type(multiList[0]) == int:
203 if type(multiList[0]) == int:
204 multiList = ast.literal_eval("(" + value + ")")
204 multiList = ast.literal_eval("(" + value + ")")
205
205
206 self.__formated_value = multiList
206 self.__formated_value = multiList
207
207
208 return self.__formated_value
208 return self.__formated_value
209
209
210 if format == 'bool':
210 if format == 'bool':
211 value = int(value)
211 value = int(value)
212
212
213 if format == 'int':
213 if format == 'int':
214 value = float(value)
214 value = float(value)
215
215
216 format_func = eval(format)
216 format_func = eval(format)
217
217
218 self.__formated_value = format_func(value)
218 self.__formated_value = format_func(value)
219
219
220 return self.__formated_value
220 return self.__formated_value
221
221
222 def updateId(self, new_id):
222 def updateId(self, new_id):
223
223
224 self.id = str(new_id)
224 self.id = str(new_id)
225
225
226 def setup(self, id, name, value, format='str'):
226 def setup(self, id, name, value, format='str'):
227 self.id = str(id)
227 self.id = str(id)
228 self.name = name
228 self.name = name
229 if format == 'obj':
229 if format == 'obj':
230 self.value = value
230 self.value = value
231 else:
231 else:
232 self.value = str(value)
232 self.value = str(value)
233 self.format = str.lower(format)
233 self.format = str.lower(format)
234
234
235 self.getValue()
235 self.getValue()
236
236
237 return 1
237 return 1
238
238
239 def update(self, name, value, format='str'):
239 def update(self, name, value, format='str'):
240
240
241 self.name = name
241 self.name = name
242 self.value = str(value)
242 self.value = str(value)
243 self.format = format
243 self.format = format
244
244
245 def makeXml(self, opElement):
245 def makeXml(self, opElement):
246 if self.name not in ('queue',):
246 if self.name not in ('queue',):
247 parmElement = SubElement(opElement, self.ELEMENTNAME)
247 parmElement = SubElement(opElement, self.ELEMENTNAME)
248 parmElement.set('id', str(self.id))
248 parmElement.set('id', str(self.id))
249 parmElement.set('name', self.name)
249 parmElement.set('name', self.name)
250 parmElement.set('value', self.value)
250 parmElement.set('value', self.value)
251 parmElement.set('format', self.format)
251 parmElement.set('format', self.format)
252
252
253 def readXml(self, parmElement):
253 def readXml(self, parmElement):
254
254
255 self.id = parmElement.get('id')
255 self.id = parmElement.get('id')
256 self.name = parmElement.get('name')
256 self.name = parmElement.get('name')
257 self.value = parmElement.get('value')
257 self.value = parmElement.get('value')
258 self.format = str.lower(parmElement.get('format'))
258 self.format = str.lower(parmElement.get('format'))
259
259
260 #Compatible with old signal chain version
260 #Compatible with old signal chain version
261 if self.format == 'int' and self.name == 'idfigure':
261 if self.format == 'int' and self.name == 'idfigure':
262 self.name = 'id'
262 self.name = 'id'
263
263
264 def printattr(self):
264 def printattr(self):
265
265
266 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
266 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
267
267
268 class OperationConf():
268 class OperationConf():
269
269
270 id = None
270 id = None
271 name = None
271 name = None
272 priority = None
272 priority = None
273 type = None
273 type = None
274
274
275 parmConfObjList = []
275 parmConfObjList = []
276
276
277 ELEMENTNAME = 'Operation'
277 ELEMENTNAME = 'Operation'
278
278
279 def __init__(self):
279 def __init__(self):
280
280
281 self.id = '0'
281 self.id = '0'
282 self.name = None
282 self.name = None
283 self.priority = None
283 self.priority = None
284 self.type = 'self'
284 self.type = 'self'
285
285
286
286
287 def __getNewId(self):
287 def __getNewId(self):
288
288
289 return int(self.id)*10 + len(self.parmConfObjList) + 1
289 return int(self.id)*10 + len(self.parmConfObjList) + 1
290
290
291 def updateId(self, new_id):
291 def updateId(self, new_id):
292
292
293 self.id = str(new_id)
293 self.id = str(new_id)
294
294
295 n = 1
295 n = 1
296 for parmObj in self.parmConfObjList:
296 for parmObj in self.parmConfObjList:
297
297
298 idParm = str(int(new_id)*10 + n)
298 idParm = str(int(new_id)*10 + n)
299 parmObj.updateId(idParm)
299 parmObj.updateId(idParm)
300
300
301 n += 1
301 n += 1
302
302
303 def getElementName(self):
303 def getElementName(self):
304
304
305 return self.ELEMENTNAME
305 return self.ELEMENTNAME
306
306
307 def getParameterObjList(self):
307 def getParameterObjList(self):
308
308
309 return self.parmConfObjList
309 return self.parmConfObjList
310
310
311 def getParameterObj(self, parameterName):
311 def getParameterObj(self, parameterName):
312
312
313 for parmConfObj in self.parmConfObjList:
313 for parmConfObj in self.parmConfObjList:
314
314
315 if parmConfObj.name != parameterName:
315 if parmConfObj.name != parameterName:
316 continue
316 continue
317
317
318 return parmConfObj
318 return parmConfObj
319
319
320 return None
320 return None
321
321
322 def getParameterObjfromValue(self, parameterValue):
322 def getParameterObjfromValue(self, parameterValue):
323
323
324 for parmConfObj in self.parmConfObjList:
324 for parmConfObj in self.parmConfObjList:
325
325
326 if parmConfObj.getValue() != parameterValue:
326 if parmConfObj.getValue() != parameterValue:
327 continue
327 continue
328
328
329 return parmConfObj.getValue()
329 return parmConfObj.getValue()
330
330
331 return None
331 return None
332
332
333 def getParameterValue(self, parameterName):
333 def getParameterValue(self, parameterName):
334
334
335 parameterObj = self.getParameterObj(parameterName)
335 parameterObj = self.getParameterObj(parameterName)
336
336
337 # if not parameterObj:
337 # if not parameterObj:
338 # return None
338 # return None
339
339
340 value = parameterObj.getValue()
340 value = parameterObj.getValue()
341
341
342 return value
342 return value
343
343
344
344
345 def getKwargs(self):
345 def getKwargs(self):
346
346
347 kwargs = {}
347 kwargs = {}
348
348
349 for parmConfObj in self.parmConfObjList:
349 for parmConfObj in self.parmConfObjList:
350 if self.name == 'run' and parmConfObj.name == 'datatype':
350 if self.name == 'run' and parmConfObj.name == 'datatype':
351 continue
351 continue
352
352
353 kwargs[parmConfObj.name] = parmConfObj.getValue()
353 kwargs[parmConfObj.name] = parmConfObj.getValue()
354
354
355 return kwargs
355 return kwargs
356
356
357 def setup(self, id, name, priority, type):
357 def setup(self, id, name, priority, type):
358
358
359 self.id = str(id)
359 self.id = str(id)
360 self.name = name
360 self.name = name
361 self.type = type
361 self.type = type
362 self.priority = priority
362 self.priority = priority
363
363
364 self.parmConfObjList = []
364 self.parmConfObjList = []
365
365
366 def removeParameters(self):
366 def removeParameters(self):
367
367
368 for obj in self.parmConfObjList:
368 for obj in self.parmConfObjList:
369 del obj
369 del obj
370
370
371 self.parmConfObjList = []
371 self.parmConfObjList = []
372
372
373 def addParameter(self, name, value, format='str'):
373 def addParameter(self, name, value, format='str'):
374
374
375 id = self.__getNewId()
375 id = self.__getNewId()
376
376
377 parmConfObj = ParameterConf()
377 parmConfObj = ParameterConf()
378 if not parmConfObj.setup(id, name, value, format):
378 if not parmConfObj.setup(id, name, value, format):
379 return None
379 return None
380
380
381 self.parmConfObjList.append(parmConfObj)
381 self.parmConfObjList.append(parmConfObj)
382
382
383 return parmConfObj
383 return parmConfObj
384
384
385 def changeParameter(self, name, value, format='str'):
385 def changeParameter(self, name, value, format='str'):
386
386
387 parmConfObj = self.getParameterObj(name)
387 parmConfObj = self.getParameterObj(name)
388 parmConfObj.update(name, value, format)
388 parmConfObj.update(name, value, format)
389
389
390 return parmConfObj
390 return parmConfObj
391
391
392 def makeXml(self, procUnitElement):
392 def makeXml(self, procUnitElement):
393
393
394 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
394 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
395 opElement.set('id', str(self.id))
395 opElement.set('id', str(self.id))
396 opElement.set('name', self.name)
396 opElement.set('name', self.name)
397 opElement.set('type', self.type)
397 opElement.set('type', self.type)
398 opElement.set('priority', str(self.priority))
398 opElement.set('priority', str(self.priority))
399
399
400 for parmConfObj in self.parmConfObjList:
400 for parmConfObj in self.parmConfObjList:
401 parmConfObj.makeXml(opElement)
401 parmConfObj.makeXml(opElement)
402
402
403 def readXml(self, opElement):
403 def readXml(self, opElement):
404
404
405 self.id = opElement.get('id')
405 self.id = opElement.get('id')
406 self.name = opElement.get('name')
406 self.name = opElement.get('name')
407 self.type = opElement.get('type')
407 self.type = opElement.get('type')
408 self.priority = opElement.get('priority')
408 self.priority = opElement.get('priority')
409
409
410 #Compatible with old signal chain version
410 #Compatible with old signal chain version
411 #Use of 'run' method instead 'init'
411 #Use of 'run' method instead 'init'
412 if self.type == 'self' and self.name == 'init':
412 if self.type == 'self' and self.name == 'init':
413 self.name = 'run'
413 self.name = 'run'
414
414
415 self.parmConfObjList = []
415 self.parmConfObjList = []
416
416
417 parmElementList = opElement.iter(ParameterConf().getElementName())
417 parmElementList = opElement.iter(ParameterConf().getElementName())
418
418
419 for parmElement in parmElementList:
419 for parmElement in parmElementList:
420 parmConfObj = ParameterConf()
420 parmConfObj = ParameterConf()
421 parmConfObj.readXml(parmElement)
421 parmConfObj.readXml(parmElement)
422
422
423 #Compatible with old signal chain version
423 #Compatible with old signal chain version
424 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
424 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
425 if self.type != 'self' and self.name == 'Plot':
425 if self.type != 'self' and self.name == 'Plot':
426 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
426 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
427 self.name = parmConfObj.value
427 self.name = parmConfObj.value
428 continue
428 continue
429
429
430 self.parmConfObjList.append(parmConfObj)
430 self.parmConfObjList.append(parmConfObj)
431
431
432 def printattr(self):
432 def printattr(self):
433
433
434 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
434 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
435 self.id,
435 self.id,
436 self.name,
436 self.name,
437 self.type,
437 self.type,
438 self.priority)
438 self.priority)
439
439
440 for parmConfObj in self.parmConfObjList:
440 for parmConfObj in self.parmConfObjList:
441 parmConfObj.printattr()
441 parmConfObj.printattr()
442
442
443 def createObject(self, plotter_queue=None):
443 def createObject(self, plotter_queue=None):
444
444
445
445
446 if self.type == 'self':
446 if self.type == 'self':
447 raise ValueError, "This operation type cannot be created"
447 raise ValueError, "This operation type cannot be created"
448
448
449 if self.type == 'plotter':
449 if self.type == 'plotter':
450 #Plotter(plotter_name)
450 #Plotter(plotter_name)
451 if not plotter_queue:
451 if not plotter_queue:
452 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
452 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
453
453
454 opObj = Plotter(self.name, plotter_queue)
454 opObj = Plotter(self.name, plotter_queue)
455
455
456 if self.type == 'external' or self.type == 'other':
456 if self.type == 'external' or self.type == 'other':
457
457
458 className = eval(self.name)
458 className = eval(self.name)
459 kwargs = self.getKwargs()
459 kwargs = self.getKwargs()
460
460
461 opObj = className(**kwargs)
461 opObj = className(**kwargs)
462
462
463 return opObj
463 return opObj
464
464
465
465
466 class ProcUnitConf():
466 class ProcUnitConf():
467
467
468 id = None
468 id = None
469 name = None
469 name = None
470 datatype = None
470 datatype = None
471 inputId = None
471 inputId = None
472 parentId = None
472 parentId = None
473
473
474 opConfObjList = []
474 opConfObjList = []
475
475
476 procUnitObj = None
476 procUnitObj = None
477 opObjList = []
477 opObjList = []
478
478
479 ELEMENTNAME = 'ProcUnit'
479 ELEMENTNAME = 'ProcUnit'
480
480
481 def __init__(self):
481 def __init__(self):
482
482
483 self.id = None
483 self.id = None
484 self.datatype = None
484 self.datatype = None
485 self.name = None
485 self.name = None
486 self.inputId = None
486 self.inputId = None
487
487
488 self.opConfObjList = []
488 self.opConfObjList = []
489
489
490 self.procUnitObj = None
490 self.procUnitObj = None
491 self.opObjDict = {}
491 self.opObjDict = {}
492
492
493 def __getPriority(self):
493 def __getPriority(self):
494
494
495 return len(self.opConfObjList)+1
495 return len(self.opConfObjList)+1
496
496
497 def __getNewId(self):
497 def __getNewId(self):
498
498
499 return int(self.id)*10 + len(self.opConfObjList) + 1
499 return int(self.id)*10 + len(self.opConfObjList) + 1
500
500
501 def getElementName(self):
501 def getElementName(self):
502
502
503 return self.ELEMENTNAME
503 return self.ELEMENTNAME
504
504
505 def getId(self):
505 def getId(self):
506
506
507 return self.id
507 return self.id
508
508
509 def updateId(self, new_id, parentId=parentId):
509 def updateId(self, new_id, parentId=parentId):
510
510
511
511
512 new_id = int(parentId)*10 + (int(self.id) % 10)
512 new_id = int(parentId)*10 + (int(self.id) % 10)
513 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
513 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
514
514
515 #If this proc unit has not inputs
515 #If this proc unit has not inputs
516 if self.inputId == '0':
516 if self.inputId == '0':
517 new_inputId = 0
517 new_inputId = 0
518
518
519 n = 1
519 n = 1
520 for opConfObj in self.opConfObjList:
520 for opConfObj in self.opConfObjList:
521
521
522 idOp = str(int(new_id)*10 + n)
522 idOp = str(int(new_id)*10 + n)
523 opConfObj.updateId(idOp)
523 opConfObj.updateId(idOp)
524
524
525 n += 1
525 n += 1
526
526
527 self.parentId = str(parentId)
527 self.parentId = str(parentId)
528 self.id = str(new_id)
528 self.id = str(new_id)
529 self.inputId = str(new_inputId)
529 self.inputId = str(new_inputId)
530
530
531
531
532 def getInputId(self):
532 def getInputId(self):
533
533
534 return self.inputId
534 return self.inputId
535
535
536 def getOperationObjList(self):
536 def getOperationObjList(self):
537
537
538 return self.opConfObjList
538 return self.opConfObjList
539
539
540 def getOperationObj(self, name=None):
540 def getOperationObj(self, name=None):
541
541
542 for opConfObj in self.opConfObjList:
542 for opConfObj in self.opConfObjList:
543
543
544 if opConfObj.name != name:
544 if opConfObj.name != name:
545 continue
545 continue
546
546
547 return opConfObj
547 return opConfObj
548
548
549 return None
549 return None
550
550
551 def getOpObjfromParamValue(self, value=None):
551 def getOpObjfromParamValue(self, value=None):
552
552
553 for opConfObj in self.opConfObjList:
553 for opConfObj in self.opConfObjList:
554 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
554 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
555 continue
555 continue
556 return opConfObj
556 return opConfObj
557 return None
557 return None
558
558
559 def getProcUnitObj(self):
559 def getProcUnitObj(self):
560
560
561 return self.procUnitObj
561 return self.procUnitObj
562
562
563 def setup(self, id, name, datatype, inputId, parentId=None):
563 def setup(self, id, name, datatype, inputId, parentId=None):
564
564
565 #Compatible with old signal chain version
565 #Compatible with old signal chain version
566 if datatype==None and name==None:
566 if datatype==None and name==None:
567 raise ValueError, "datatype or name should be defined"
567 raise ValueError, "datatype or name should be defined"
568
568
569 if name==None:
569 if name==None:
570 if 'Proc' in datatype:
570 if 'Proc' in datatype:
571 name = datatype
571 name = datatype
572 else:
572 else:
573 name = '%sProc' %(datatype)
573 name = '%sProc' %(datatype)
574
574
575 if datatype==None:
575 if datatype==None:
576 datatype = name.replace('Proc','')
576 datatype = name.replace('Proc','')
577
577
578 self.id = str(id)
578 self.id = str(id)
579 self.name = name
579 self.name = name
580 self.datatype = datatype
580 self.datatype = datatype
581 self.inputId = inputId
581 self.inputId = inputId
582 self.parentId = parentId
582 self.parentId = parentId
583
583
584 self.opConfObjList = []
584 self.opConfObjList = []
585
585
586 self.addOperation(name='run', optype='self')
586 self.addOperation(name='run', optype='self')
587
587
588 def removeOperations(self):
588 def removeOperations(self):
589
589
590 for obj in self.opConfObjList:
590 for obj in self.opConfObjList:
591 del obj
591 del obj
592
592
593 self.opConfObjList = []
593 self.opConfObjList = []
594 self.addOperation(name='run')
594 self.addOperation(name='run')
595
595
596 def addParameter(self, **kwargs):
596 def addParameter(self, **kwargs):
597 '''
597 '''
598 Add parameters to "run" operation
598 Add parameters to "run" operation
599 '''
599 '''
600 opObj = self.opConfObjList[0]
600 opObj = self.opConfObjList[0]
601
601
602 opObj.addParameter(**kwargs)
602 opObj.addParameter(**kwargs)
603
603
604 return opObj
604 return opObj
605
605
606 def addOperation(self, name, optype='self'):
606 def addOperation(self, name, optype='self'):
607
607
608 id = self.__getNewId()
608 id = self.__getNewId()
609 priority = self.__getPriority()
609 priority = self.__getPriority()
610
610
611 opConfObj = OperationConf()
611 opConfObj = OperationConf()
612 opConfObj.setup(id, name=name, priority=priority, type=optype)
612 opConfObj.setup(id, name=name, priority=priority, type=optype)
613
613
614 self.opConfObjList.append(opConfObj)
614 self.opConfObjList.append(opConfObj)
615
615
616 return opConfObj
616 return opConfObj
617
617
618 def makeXml(self, projectElement):
618 def makeXml(self, projectElement):
619
619
620 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
620 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
621 procUnitElement.set('id', str(self.id))
621 procUnitElement.set('id', str(self.id))
622 procUnitElement.set('name', self.name)
622 procUnitElement.set('name', self.name)
623 procUnitElement.set('datatype', self.datatype)
623 procUnitElement.set('datatype', self.datatype)
624 procUnitElement.set('inputId', str(self.inputId))
624 procUnitElement.set('inputId', str(self.inputId))
625
625
626 for opConfObj in self.opConfObjList:
626 for opConfObj in self.opConfObjList:
627 opConfObj.makeXml(procUnitElement)
627 opConfObj.makeXml(procUnitElement)
628
628
629 def readXml(self, upElement):
629 def readXml(self, upElement):
630
630
631 self.id = upElement.get('id')
631 self.id = upElement.get('id')
632 self.name = upElement.get('name')
632 self.name = upElement.get('name')
633 self.datatype = upElement.get('datatype')
633 self.datatype = upElement.get('datatype')
634 self.inputId = upElement.get('inputId')
634 self.inputId = upElement.get('inputId')
635
635
636 if self.ELEMENTNAME == "ReadUnit":
636 if self.ELEMENTNAME == "ReadUnit":
637 self.datatype = self.datatype.replace("Reader", "")
637 self.datatype = self.datatype.replace("Reader", "")
638
638
639 if self.ELEMENTNAME == "ProcUnit":
639 if self.ELEMENTNAME == "ProcUnit":
640 self.datatype = self.datatype.replace("Proc", "")
640 self.datatype = self.datatype.replace("Proc", "")
641
641
642 if self.inputId == 'None':
642 if self.inputId == 'None':
643 self.inputId = '0'
643 self.inputId = '0'
644
644
645 self.opConfObjList = []
645 self.opConfObjList = []
646
646
647 opElementList = upElement.iter(OperationConf().getElementName())
647 opElementList = upElement.iter(OperationConf().getElementName())
648
648
649 for opElement in opElementList:
649 for opElement in opElementList:
650 opConfObj = OperationConf()
650 opConfObj = OperationConf()
651 opConfObj.readXml(opElement)
651 opConfObj.readXml(opElement)
652 self.opConfObjList.append(opConfObj)
652 self.opConfObjList.append(opConfObj)
653
653
654 def printattr(self):
654 def printattr(self):
655
655
656 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
656 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
657 self.id,
657 self.id,
658 self.name,
658 self.name,
659 self.datatype,
659 self.datatype,
660 self.inputId)
660 self.inputId)
661
661
662 for opConfObj in self.opConfObjList:
662 for opConfObj in self.opConfObjList:
663 opConfObj.printattr()
663 opConfObj.printattr()
664
664
665
665
666 def getKwargs(self):
666 def getKwargs(self):
667
667
668 opObj = self.opConfObjList[0]
668 opObj = self.opConfObjList[0]
669 kwargs = opObj.getKwargs()
669 kwargs = opObj.getKwargs()
670
670
671 return kwargs
671 return kwargs
672
672
673 def createObjects(self, plotter_queue=None):
673 def createObjects(self, plotter_queue=None):
674
674
675 className = eval(self.name)
675 className = eval(self.name)
676 kwargs = self.getKwargs()
676 kwargs = self.getKwargs()
677 procUnitObj = className(**kwargs)
677 procUnitObj = className(**kwargs)
678
678
679 for opConfObj in self.opConfObjList:
679 for opConfObj in self.opConfObjList:
680
680
681 if opConfObj.type=='self' and self.name=='run':
681 if opConfObj.type=='self' and self.name=='run':
682 continue
682 continue
683 elif opConfObj.type=='self':
683 elif opConfObj.type=='self':
684 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
684 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
685 continue
685 continue
686
686
687 opObj = opConfObj.createObject(plotter_queue)
687 opObj = opConfObj.createObject(plotter_queue)
688
688
689 self.opObjDict[opConfObj.id] = opObj
689 self.opObjDict[opConfObj.id] = opObj
690
690
691 procUnitObj.addOperation(opObj, opConfObj.id)
691 procUnitObj.addOperation(opObj, opConfObj.id)
692
692
693 self.procUnitObj = procUnitObj
693 self.procUnitObj = procUnitObj
694
694
695 return procUnitObj
695 return procUnitObj
696
696
697 ## @profile
697 ## @profile
698 def run(self):
698 def run(self):
699
699
700 is_ok = False
700 is_ok = False
701
701
702 for opConfObj in self.opConfObjList:
702 for opConfObj in self.opConfObjList:
703
703
704 kwargs = {}
704 kwargs = {}
705 for parmConfObj in opConfObj.getParameterObjList():
705 for parmConfObj in opConfObj.getParameterObjList():
706 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
706 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
707 continue
707 continue
708
708
709 kwargs[parmConfObj.name] = parmConfObj.getValue()
709 kwargs[parmConfObj.name] = parmConfObj.getValue()
710
710
711 #ini = time.time()
711 #ini = time.time()
712
712
713 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
713 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
714 sts = self.procUnitObj.call(opType = opConfObj.type,
714 sts = self.procUnitObj.call(opType = opConfObj.type,
715 opName = opConfObj.name,
715 opName = opConfObj.name,
716 opId = opConfObj.id,
716 opId = opConfObj.id,
717 )
717 **kwargs)
718
718
719 # total_time = time.time() - ini
719 # total_time = time.time() - ini
720 #
720 #
721 # if total_time > 0.002:
721 # if total_time > 0.002:
722 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
722 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
723
723
724 is_ok = is_ok or sts
724 is_ok = is_ok or sts
725
725
726 return is_ok
726 return is_ok
727
727
728 def close(self):
728 def close(self):
729
729
730 for opConfObj in self.opConfObjList:
730 for opConfObj in self.opConfObjList:
731 if opConfObj.type == 'self':
731 if opConfObj.type == 'self':
732 continue
732 continue
733
733
734 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
734 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
735 opObj.close()
735 opObj.close()
736
736
737 self.procUnitObj.close()
737 self.procUnitObj.close()
738
738
739 return
739 return
740
740
741 class ReadUnitConf(ProcUnitConf):
741 class ReadUnitConf(ProcUnitConf):
742
742
743 path = None
743 path = None
744 startDate = None
744 startDate = None
745 endDate = None
745 endDate = None
746 startTime = None
746 startTime = None
747 endTime = None
747 endTime = None
748
748
749 ELEMENTNAME = 'ReadUnit'
749 ELEMENTNAME = 'ReadUnit'
750
750
751 def __init__(self):
751 def __init__(self):
752
752
753 self.id = None
753 self.id = None
754 self.datatype = None
754 self.datatype = None
755 self.name = None
755 self.name = None
756 self.inputId = None
756 self.inputId = None
757
757
758 self.parentId = None
758 self.parentId = None
759
759
760 self.opConfObjList = []
760 self.opConfObjList = []
761 self.opObjList = []
761 self.opObjList = []
762
762
763 def getElementName(self):
763 def getElementName(self):
764
764
765 return self.ELEMENTNAME
765 return self.ELEMENTNAME
766
766
767 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
767 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
768 endTime="", parentId=None, queue=None, server=None, **kwargs):
768 endTime="", parentId=None, queue=None, server=None, **kwargs):
769
769
770 #Compatible with old signal chain version
770 #Compatible with old signal chain version
771 if datatype==None and name==None:
771 if datatype==None and name==None:
772 raise ValueError, "datatype or name should be defined"
772 raise ValueError, "datatype or name should be defined"
773
773
774 if name==None:
774 if name==None:
775 if 'Reader' in datatype:
775 if 'Reader' in datatype:
776 name = datatype
776 name = datatype
777 else:
777 else:
778 name = '%sReader' %(datatype)
778 name = '%sReader' %(datatype)
779 if datatype==None:
779 if datatype==None:
780 datatype = name.replace('Reader','')
780 datatype = name.replace('Reader','')
781
781
782 self.id = id
782 self.id = id
783 self.name = name
783 self.name = name
784 self.datatype = datatype
784 self.datatype = datatype
785 if path != '':
785 if path != '':
786 self.path = os.path.abspath(path)
786 self.path = os.path.abspath(path)
787 self.startDate = startDate
787 self.startDate = startDate
788 self.endDate = endDate
788 self.endDate = endDate
789 self.startTime = startTime
789 self.startTime = startTime
790 self.endTime = endTime
790 self.endTime = endTime
791
791
792 self.inputId = '0'
792 self.inputId = '0'
793 self.parentId = parentId
793 self.parentId = parentId
794 self.queue = queue
794 self.queue = queue
795 self.server = server
795 self.server = server
796 self.addRunOperation(**kwargs)
796 self.addRunOperation(**kwargs)
797
797
798 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
798 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
799
799
800 #Compatible with old signal chain version
800 #Compatible with old signal chain version
801 if datatype==None and name==None:
801 if datatype==None and name==None:
802 raise ValueError, "datatype or name should be defined"
802 raise ValueError, "datatype or name should be defined"
803
803
804 if name==None:
804 if name==None:
805 if 'Reader' in datatype:
805 if 'Reader' in datatype:
806 name = datatype
806 name = datatype
807 else:
807 else:
808 name = '%sReader' %(datatype)
808 name = '%sReader' %(datatype)
809
809
810 if datatype==None:
810 if datatype==None:
811 datatype = name.replace('Reader','')
811 datatype = name.replace('Reader','')
812
812
813 self.datatype = datatype
813 self.datatype = datatype
814 self.name = name
814 self.name = name
815 self.path = path
815 self.path = path
816 self.startDate = startDate
816 self.startDate = startDate
817 self.endDate = endDate
817 self.endDate = endDate
818 self.startTime = startTime
818 self.startTime = startTime
819 self.endTime = endTime
819 self.endTime = endTime
820
820
821 self.inputId = '0'
821 self.inputId = '0'
822 self.parentId = parentId
822 self.parentId = parentId
823
823
824 self.updateRunOperation(**kwargs)
824 self.updateRunOperation(**kwargs)
825
825
826 def removeOperations(self):
826 def removeOperations(self):
827
827
828 for obj in self.opConfObjList:
828 for obj in self.opConfObjList:
829 del obj
829 del obj
830
830
831 self.opConfObjList = []
831 self.opConfObjList = []
832
832
833 def addRunOperation(self, **kwargs):
833 def addRunOperation(self, **kwargs):
834
834
835 opObj = self.addOperation(name = 'run', optype = 'self')
835 opObj = self.addOperation(name = 'run', optype = 'self')
836
836
837 if self.server is None:
837 if self.server is None:
838 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
838 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
839 opObj.addParameter(name='path' , value=self.path, format='str')
839 opObj.addParameter(name='path' , value=self.path, format='str')
840 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
840 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
841 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
841 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
842 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
842 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
843 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
843 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
844 opObj.addParameter(name='queue' , value=self.queue, format='obj')
844 opObj.addParameter(name='queue' , value=self.queue, format='obj')
845 for key, value in kwargs.items():
845 for key, value in kwargs.items():
846 opObj.addParameter(name=key, value=value, format=type(value).__name__)
846 opObj.addParameter(name=key, value=value, format=type(value).__name__)
847 else:
847 else:
848 opObj.addParameter(name='server' , value=self.server, format='str')
848 opObj.addParameter(name='server' , value=self.server, format='str')
849
849
850
850
851 return opObj
851 return opObj
852
852
853 def updateRunOperation(self, **kwargs):
853 def updateRunOperation(self, **kwargs):
854
854
855 opObj = self.getOperationObj(name = 'run')
855 opObj = self.getOperationObj(name = 'run')
856 opObj.removeParameters()
856 opObj.removeParameters()
857
857
858 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
858 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
859 opObj.addParameter(name='path' , value=self.path, format='str')
859 opObj.addParameter(name='path' , value=self.path, format='str')
860 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
860 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
861 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
861 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
862 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
862 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
863 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
863 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
864
864
865 for key, value in kwargs.items():
865 for key, value in kwargs.items():
866 opObj.addParameter(name=key, value=value, format=type(value).__name__)
866 opObj.addParameter(name=key, value=value, format=type(value).__name__)
867
867
868 return opObj
868 return opObj
869
869
870 # def makeXml(self, projectElement):
870 # def makeXml(self, projectElement):
871 #
871 #
872 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
872 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
873 # procUnitElement.set('id', str(self.id))
873 # procUnitElement.set('id', str(self.id))
874 # procUnitElement.set('name', self.name)
874 # procUnitElement.set('name', self.name)
875 # procUnitElement.set('datatype', self.datatype)
875 # procUnitElement.set('datatype', self.datatype)
876 # procUnitElement.set('inputId', str(self.inputId))
876 # procUnitElement.set('inputId', str(self.inputId))
877 #
877 #
878 # for opConfObj in self.opConfObjList:
878 # for opConfObj in self.opConfObjList:
879 # opConfObj.makeXml(procUnitElement)
879 # opConfObj.makeXml(procUnitElement)
880
880
881 def readXml(self, upElement):
881 def readXml(self, upElement):
882
882
883 self.id = upElement.get('id')
883 self.id = upElement.get('id')
884 self.name = upElement.get('name')
884 self.name = upElement.get('name')
885 self.datatype = upElement.get('datatype')
885 self.datatype = upElement.get('datatype')
886 self.inputId = upElement.get('inputId')
886 self.inputId = upElement.get('inputId')
887
887
888 if self.ELEMENTNAME == "ReadUnit":
888 if self.ELEMENTNAME == "ReadUnit":
889 self.datatype = self.datatype.replace("Reader", "")
889 self.datatype = self.datatype.replace("Reader", "")
890
890
891 if self.inputId == 'None':
891 if self.inputId == 'None':
892 self.inputId = '0'
892 self.inputId = '0'
893
893
894 self.opConfObjList = []
894 self.opConfObjList = []
895
895
896 opElementList = upElement.iter(OperationConf().getElementName())
896 opElementList = upElement.iter(OperationConf().getElementName())
897
897
898 for opElement in opElementList:
898 for opElement in opElementList:
899 opConfObj = OperationConf()
899 opConfObj = OperationConf()
900 opConfObj.readXml(opElement)
900 opConfObj.readXml(opElement)
901 self.opConfObjList.append(opConfObj)
901 self.opConfObjList.append(opConfObj)
902
902
903 if opConfObj.name == 'run':
903 if opConfObj.name == 'run':
904 self.path = opConfObj.getParameterValue('path')
904 self.path = opConfObj.getParameterValue('path')
905 self.startDate = opConfObj.getParameterValue('startDate')
905 self.startDate = opConfObj.getParameterValue('startDate')
906 self.endDate = opConfObj.getParameterValue('endDate')
906 self.endDate = opConfObj.getParameterValue('endDate')
907 self.startTime = opConfObj.getParameterValue('startTime')
907 self.startTime = opConfObj.getParameterValue('startTime')
908 self.endTime = opConfObj.getParameterValue('endTime')
908 self.endTime = opConfObj.getParameterValue('endTime')
909
909
910 class Project():
910 class Project():
911
911
912 id = None
912 id = None
913 name = None
913 name = None
914 description = None
914 description = None
915 filename = None
915 filename = None
916
916
917 procUnitConfObjDict = None
917 procUnitConfObjDict = None
918
918
919 ELEMENTNAME = 'Project'
919 ELEMENTNAME = 'Project'
920
920
921 plotterQueue = None
921 plotterQueue = None
922
922
923 def __init__(self, plotter_queue=None):
923 def __init__(self, plotter_queue=None):
924
924
925 self.id = None
925 self.id = None
926 self.name = None
926 self.name = None
927 self.description = None
927 self.description = None
928
928
929 self.plotterQueue = plotter_queue
929 self.plotterQueue = plotter_queue
930
930
931 self.procUnitConfObjDict = {}
931 self.procUnitConfObjDict = {}
932
932
933 def __getNewId(self):
933 def __getNewId(self):
934
934
935 idList = self.procUnitConfObjDict.keys()
935 idList = self.procUnitConfObjDict.keys()
936
936
937 id = int(self.id)*10
937 id = int(self.id)*10
938
938
939 while True:
939 while True:
940 id += 1
940 id += 1
941
941
942 if str(id) in idList:
942 if str(id) in idList:
943 continue
943 continue
944
944
945 break
945 break
946
946
947 return str(id)
947 return str(id)
948
948
949 def getElementName(self):
949 def getElementName(self):
950
950
951 return self.ELEMENTNAME
951 return self.ELEMENTNAME
952
952
953 def getId(self):
953 def getId(self):
954
954
955 return self.id
955 return self.id
956
956
957 def updateId(self, new_id):
957 def updateId(self, new_id):
958
958
959 self.id = str(new_id)
959 self.id = str(new_id)
960
960
961 keyList = self.procUnitConfObjDict.keys()
961 keyList = self.procUnitConfObjDict.keys()
962 keyList.sort()
962 keyList.sort()
963
963
964 n = 1
964 n = 1
965 newProcUnitConfObjDict = {}
965 newProcUnitConfObjDict = {}
966
966
967 for procKey in keyList:
967 for procKey in keyList:
968
968
969 procUnitConfObj = self.procUnitConfObjDict[procKey]
969 procUnitConfObj = self.procUnitConfObjDict[procKey]
970 idProcUnit = str(int(self.id)*10 + n)
970 idProcUnit = str(int(self.id)*10 + n)
971 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
971 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
972
972
973 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
973 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
974 n += 1
974 n += 1
975
975
976 self.procUnitConfObjDict = newProcUnitConfObjDict
976 self.procUnitConfObjDict = newProcUnitConfObjDict
977
977
978 def setup(self, id, name, description):
978 def setup(self, id, name, description):
979
979
980 self.id = str(id)
980 self.id = str(id)
981 self.name = name
981 self.name = name
982 self.description = description
982 self.description = description
983
983
984 def update(self, name, description):
984 def update(self, name, description):
985
985
986 self.name = name
986 self.name = name
987 self.description = description
987 self.description = description
988
988
989 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
989 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
990
990
991 if id is None:
991 if id is None:
992 idReadUnit = self.__getNewId()
992 idReadUnit = self.__getNewId()
993 else:
993 else:
994 idReadUnit = str(id)
994 idReadUnit = str(id)
995
995
996 readUnitConfObj = ReadUnitConf()
996 readUnitConfObj = ReadUnitConf()
997 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
997 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
998
998
999 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
999 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1000
1000
1001 return readUnitConfObj
1001 return readUnitConfObj
1002
1002
1003 def addProcUnit(self, inputId='0', datatype=None, name=None):
1003 def addProcUnit(self, inputId='0', datatype=None, name=None):
1004
1004
1005 idProcUnit = self.__getNewId()
1005 idProcUnit = self.__getNewId()
1006
1006
1007 procUnitConfObj = ProcUnitConf()
1007 procUnitConfObj = ProcUnitConf()
1008 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1008 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1009
1009
1010 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1010 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1011
1011
1012 return procUnitConfObj
1012 return procUnitConfObj
1013
1013
1014 def removeProcUnit(self, id):
1014 def removeProcUnit(self, id):
1015
1015
1016 if id in self.procUnitConfObjDict.keys():
1016 if id in self.procUnitConfObjDict.keys():
1017 self.procUnitConfObjDict.pop(id)
1017 self.procUnitConfObjDict.pop(id)
1018
1018
1019 def getReadUnitId(self):
1019 def getReadUnitId(self):
1020
1020
1021 readUnitConfObj = self.getReadUnitObj()
1021 readUnitConfObj = self.getReadUnitObj()
1022
1022
1023 return readUnitConfObj.id
1023 return readUnitConfObj.id
1024
1024
1025 def getReadUnitObj(self):
1025 def getReadUnitObj(self):
1026
1026
1027 for obj in self.procUnitConfObjDict.values():
1027 for obj in self.procUnitConfObjDict.values():
1028 if obj.getElementName() == "ReadUnit":
1028 if obj.getElementName() == "ReadUnit":
1029 return obj
1029 return obj
1030
1030
1031 return None
1031 return None
1032
1032
1033 def getProcUnitObj(self, id=None, name=None):
1033 def getProcUnitObj(self, id=None, name=None):
1034
1034
1035 if id != None:
1035 if id != None:
1036 return self.procUnitConfObjDict[id]
1036 return self.procUnitConfObjDict[id]
1037
1037
1038 if name != None:
1038 if name != None:
1039 return self.getProcUnitObjByName(name)
1039 return self.getProcUnitObjByName(name)
1040
1040
1041 return None
1041 return None
1042
1042
1043 def getProcUnitObjByName(self, name):
1043 def getProcUnitObjByName(self, name):
1044
1044
1045 for obj in self.procUnitConfObjDict.values():
1045 for obj in self.procUnitConfObjDict.values():
1046 if obj.name == name:
1046 if obj.name == name:
1047 return obj
1047 return obj
1048
1048
1049 return None
1049 return None
1050
1050
1051 def procUnitItems(self):
1051 def procUnitItems(self):
1052
1052
1053 return self.procUnitConfObjDict.items()
1053 return self.procUnitConfObjDict.items()
1054
1054
1055 def makeXml(self):
1055 def makeXml(self):
1056
1056
1057 projectElement = Element('Project')
1057 projectElement = Element('Project')
1058 projectElement.set('id', str(self.id))
1058 projectElement.set('id', str(self.id))
1059 projectElement.set('name', self.name)
1059 projectElement.set('name', self.name)
1060 projectElement.set('description', self.description)
1060 projectElement.set('description', self.description)
1061
1061
1062 for procUnitConfObj in self.procUnitConfObjDict.values():
1062 for procUnitConfObj in self.procUnitConfObjDict.values():
1063 procUnitConfObj.makeXml(projectElement)
1063 procUnitConfObj.makeXml(projectElement)
1064
1064
1065 self.projectElement = projectElement
1065 self.projectElement = projectElement
1066
1066
1067 def writeXml(self, filename=None):
1067 def writeXml(self, filename=None):
1068
1068
1069 if filename == None:
1069 if filename == None:
1070 if self.filename:
1070 if self.filename:
1071 filename = self.filename
1071 filename = self.filename
1072 else:
1072 else:
1073 filename = "schain.xml"
1073 filename = "schain.xml"
1074
1074
1075 if not filename:
1075 if not filename:
1076 print "filename has not been defined. Use setFilename(filename) for do it."
1076 print "filename has not been defined. Use setFilename(filename) for do it."
1077 return 0
1077 return 0
1078
1078
1079 abs_file = os.path.abspath(filename)
1079 abs_file = os.path.abspath(filename)
1080
1080
1081 if not os.access(os.path.dirname(abs_file), os.W_OK):
1081 if not os.access(os.path.dirname(abs_file), os.W_OK):
1082 print "No write permission on %s" %os.path.dirname(abs_file)
1082 print "No write permission on %s" %os.path.dirname(abs_file)
1083 return 0
1083 return 0
1084
1084
1085 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1085 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1086 print "File %s already exists and it could not be overwriten" %abs_file
1086 print "File %s already exists and it could not be overwriten" %abs_file
1087 return 0
1087 return 0
1088
1088
1089 self.makeXml()
1089 self.makeXml()
1090
1090
1091 ElementTree(self.projectElement).write(abs_file, method='xml')
1091 ElementTree(self.projectElement).write(abs_file, method='xml')
1092
1092
1093 self.filename = abs_file
1093 self.filename = abs_file
1094
1094
1095 return 1
1095 return 1
1096
1096
1097 def readXml(self, filename = None):
1097 def readXml(self, filename = None):
1098
1098
1099 if not filename:
1099 if not filename:
1100 print "filename is not defined"
1100 print "filename is not defined"
1101 return 0
1101 return 0
1102
1102
1103 abs_file = os.path.abspath(filename)
1103 abs_file = os.path.abspath(filename)
1104
1104
1105 if not os.path.isfile(abs_file):
1105 if not os.path.isfile(abs_file):
1106 print "%s file does not exist" %abs_file
1106 print "%s file does not exist" %abs_file
1107 return 0
1107 return 0
1108
1108
1109 self.projectElement = None
1109 self.projectElement = None
1110 self.procUnitConfObjDict = {}
1110 self.procUnitConfObjDict = {}
1111
1111
1112 try:
1112 try:
1113 self.projectElement = ElementTree().parse(abs_file)
1113 self.projectElement = ElementTree().parse(abs_file)
1114 except:
1114 except:
1115 print "Error reading %s, verify file format" %filename
1115 print "Error reading %s, verify file format" %filename
1116 return 0
1116 return 0
1117
1117
1118 self.project = self.projectElement.tag
1118 self.project = self.projectElement.tag
1119
1119
1120 self.id = self.projectElement.get('id')
1120 self.id = self.projectElement.get('id')
1121 self.name = self.projectElement.get('name')
1121 self.name = self.projectElement.get('name')
1122 self.description = self.projectElement.get('description')
1122 self.description = self.projectElement.get('description')
1123
1123
1124 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1124 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1125
1125
1126 for readUnitElement in readUnitElementList:
1126 for readUnitElement in readUnitElementList:
1127 readUnitConfObj = ReadUnitConf()
1127 readUnitConfObj = ReadUnitConf()
1128 readUnitConfObj.readXml(readUnitElement)
1128 readUnitConfObj.readXml(readUnitElement)
1129
1129
1130 if readUnitConfObj.parentId == None:
1130 if readUnitConfObj.parentId == None:
1131 readUnitConfObj.parentId = self.id
1131 readUnitConfObj.parentId = self.id
1132
1132
1133 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1133 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1134
1134
1135 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1135 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1136
1136
1137 for procUnitElement in procUnitElementList:
1137 for procUnitElement in procUnitElementList:
1138 procUnitConfObj = ProcUnitConf()
1138 procUnitConfObj = ProcUnitConf()
1139 procUnitConfObj.readXml(procUnitElement)
1139 procUnitConfObj.readXml(procUnitElement)
1140
1140
1141 if procUnitConfObj.parentId == None:
1141 if procUnitConfObj.parentId == None:
1142 procUnitConfObj.parentId = self.id
1142 procUnitConfObj.parentId = self.id
1143
1143
1144 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1144 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1145
1145
1146 self.filename = abs_file
1146 self.filename = abs_file
1147
1147
1148 return 1
1148 return 1
1149
1149
1150 def printattr(self):
1150 def printattr(self):
1151
1151
1152 print "Project[%s]: name = %s, description = %s" %(self.id,
1152 print "Project[%s]: name = %s, description = %s" %(self.id,
1153 self.name,
1153 self.name,
1154 self.description)
1154 self.description)
1155
1155
1156 for procUnitConfObj in self.procUnitConfObjDict.values():
1156 for procUnitConfObj in self.procUnitConfObjDict.values():
1157 procUnitConfObj.printattr()
1157 procUnitConfObj.printattr()
1158
1158
1159 def createObjects(self):
1159 def createObjects(self):
1160
1160
1161 for procUnitConfObj in self.procUnitConfObjDict.values():
1161 for procUnitConfObj in self.procUnitConfObjDict.values():
1162 procUnitConfObj.createObjects(self.plotterQueue)
1162 procUnitConfObj.createObjects(self.plotterQueue)
1163
1163
1164 def __connect(self, objIN, thisObj):
1164 def __connect(self, objIN, thisObj):
1165
1165
1166 thisObj.setInput(objIN.getOutputObj())
1166 thisObj.setInput(objIN.getOutputObj())
1167
1167
1168 def connectObjects(self):
1168 def connectObjects(self):
1169
1169
1170 for thisPUConfObj in self.procUnitConfObjDict.values():
1170 for thisPUConfObj in self.procUnitConfObjDict.values():
1171
1171
1172 inputId = thisPUConfObj.getInputId()
1172 inputId = thisPUConfObj.getInputId()
1173
1173
1174 if int(inputId) == 0:
1174 if int(inputId) == 0:
1175 continue
1175 continue
1176
1176
1177 #Get input object
1177 #Get input object
1178 puConfINObj = self.procUnitConfObjDict[inputId]
1178 puConfINObj = self.procUnitConfObjDict[inputId]
1179 puObjIN = puConfINObj.getProcUnitObj()
1179 puObjIN = puConfINObj.getProcUnitObj()
1180
1180
1181 #Get current object
1181 #Get current object
1182 thisPUObj = thisPUConfObj.getProcUnitObj()
1182 thisPUObj = thisPUConfObj.getProcUnitObj()
1183
1183
1184 self.__connect(puObjIN, thisPUObj)
1184 self.__connect(puObjIN, thisPUObj)
1185
1185
1186 def __handleError(self, procUnitConfObj, send_email=True):
1186 def __handleError(self, procUnitConfObj, send_email=True):
1187
1187
1188 import socket
1188 import socket
1189
1189
1190 err = traceback.format_exception(sys.exc_info()[0],
1190 err = traceback.format_exception(sys.exc_info()[0],
1191 sys.exc_info()[1],
1191 sys.exc_info()[1],
1192 sys.exc_info()[2])
1192 sys.exc_info()[2])
1193
1193
1194 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1194 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1195 print "***** %s" %err[-1]
1195 print "***** %s" %err[-1]
1196
1196
1197 message = "".join(err)
1197 message = "".join(err)
1198
1198
1199 sys.stderr.write(message)
1199 sys.stderr.write(message)
1200
1200
1201 if not send_email:
1201 if not send_email:
1202 return
1202 return
1203
1203
1204 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1204 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1205
1205
1206 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1206 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1207 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1207 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1208 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1208 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1209 subtitle += "Configuration file: %s\n" %self.filename
1209 subtitle += "Configuration file: %s\n" %self.filename
1210 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1210 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1211
1211
1212 readUnitConfObj = self.getReadUnitObj()
1212 readUnitConfObj = self.getReadUnitObj()
1213 if readUnitConfObj:
1213 if readUnitConfObj:
1214 subtitle += "\nInput parameters:\n"
1214 subtitle += "\nInput parameters:\n"
1215 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1215 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1216 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1216 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1217 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1217 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1218 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1218 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1219 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1219 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1220 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1220 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1221
1221
1222 adminObj = schainpy.admin.SchainNotify()
1222 adminObj = schainpy.admin.SchainNotify()
1223 adminObj.sendAlert(message=message,
1223 adminObj.sendAlert(message=message,
1224 subject=subject,
1224 subject=subject,
1225 subtitle=subtitle,
1225 subtitle=subtitle,
1226 filename=self.filename)
1226 filename=self.filename)
1227
1227
1228 def isPaused(self):
1228 def isPaused(self):
1229 return 0
1229 return 0
1230
1230
1231 def isStopped(self):
1231 def isStopped(self):
1232 return 0
1232 return 0
1233
1233
1234 def runController(self):
1234 def runController(self):
1235 """
1235 """
1236 returns 0 when this process has been stopped, 1 otherwise
1236 returns 0 when this process has been stopped, 1 otherwise
1237 """
1237 """
1238
1238
1239 if self.isPaused():
1239 if self.isPaused():
1240 print "Process suspended"
1240 print "Process suspended"
1241
1241
1242 while True:
1242 while True:
1243 sleep(0.1)
1243 sleep(0.1)
1244
1244
1245 if not self.isPaused():
1245 if not self.isPaused():
1246 break
1246 break
1247
1247
1248 if self.isStopped():
1248 if self.isStopped():
1249 break
1249 break
1250
1250
1251 print "Process reinitialized"
1251 print "Process reinitialized"
1252
1252
1253 if self.isStopped():
1253 if self.isStopped():
1254 print "Process stopped"
1254 print "Process stopped"
1255 return 0
1255 return 0
1256
1256
1257 return 1
1257 return 1
1258
1258
1259 def setFilename(self, filename):
1259 def setFilename(self, filename):
1260
1260
1261 self.filename = filename
1261 self.filename = filename
1262
1262
1263 def setPlotterQueue(self, plotter_queue):
1263 def setPlotterQueue(self, plotter_queue):
1264
1264
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1266
1266
1267 def getPlotterQueue(self):
1267 def getPlotterQueue(self):
1268
1268
1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1270
1270
1271 def useExternalPlotter(self):
1271 def useExternalPlotter(self):
1272
1272
1273 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1273 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1274
1274
1275
1275
1276 def run(self):
1276 def run(self):
1277
1277
1278 print
1278 print
1279 print "*"*60
1279 print "*"*60
1280 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1280 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1281 print "*"*60
1281 print "*"*60
1282 print
1282 print
1283
1283
1284 keyList = self.procUnitConfObjDict.keys()
1284 keyList = self.procUnitConfObjDict.keys()
1285 keyList.sort()
1285 keyList.sort()
1286
1286
1287 while(True):
1287 while(True):
1288
1288
1289 is_ok = False
1289 is_ok = False
1290
1290
1291 for procKey in keyList:
1291 for procKey in keyList:
1292 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1292 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1293
1293
1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295
1295
1296 try:
1296 try:
1297 sts = procUnitConfObj.run()
1297 sts = procUnitConfObj.run()
1298 is_ok = is_ok or sts
1298 is_ok = is_ok or sts
1299 except KeyboardInterrupt:
1299 except KeyboardInterrupt:
1300 is_ok = False
1300 is_ok = False
1301 break
1301 break
1302 except ValueError, e:
1302 except ValueError, e:
1303 sleep(0.5)
1303 sleep(0.5)
1304 self.__handleError(procUnitConfObj, send_email=True)
1304 self.__handleError(procUnitConfObj, send_email=True)
1305 is_ok = False
1305 is_ok = False
1306 break
1306 break
1307 except:
1307 except:
1308 sleep(0.5)
1308 sleep(0.5)
1309 self.__handleError(procUnitConfObj)
1309 self.__handleError(procUnitConfObj)
1310 is_ok = False
1310 is_ok = False
1311 break
1311 break
1312
1312
1313 #If every process unit finished so end process
1313 #If every process unit finished so end process
1314 if not(is_ok):
1314 if not(is_ok):
1315 # print "Every process unit have finished"
1315 # print "Every process unit have finished"
1316 break
1316 break
1317
1317
1318 if not self.runController():
1318 if not self.runController():
1319 break
1319 break
1320
1320
1321 #Closing every process
1321 #Closing every process
1322 for procKey in keyList:
1322 for procKey in keyList:
1323 procUnitConfObj = self.procUnitConfObjDict[procKey]
1323 procUnitConfObj = self.procUnitConfObjDict[procKey]
1324 procUnitConfObj.close()
1324 procUnitConfObj.close()
1325
1325
1326 print "Process finished"
1326 print "Process finished"
1327
1327
1328 def start(self, filename=None):
1328 def start(self, filename=None):
1329
1329
1330 self.writeXml(filename)
1330 self.writeXml(filename)
1331 self.createObjects()
1331 self.createObjects()
1332 self.connectObjects()
1332 self.connectObjects()
1333 self.run()
1333 self.run()
Show file before | Show file after | Hide comments
@@ -1,179 +1,179
1 import threading
1 import threading
2 from Queue import Queue
2 from Queue import Queue
3
3
4 from schainpy.controller import Project
4 from schainpy.controller import Project
5 from schainpy.model.graphics.jroplotter import PlotManager
5 from schainpy.model.graphics.jroplotter import PlotManager
6
6
7 class ControllerThread(threading.Thread, Project):
7 class ControllerThread(threading.Thread, Project):
8
8
9 def __init__(self, plotter_queue=None):
9 def __init__(self, plotter_queue=None):
10
10
11 threading.Thread.__init__(self)
11 threading.Thread.__init__(self)
12 Project.__init__(self, plotter_queue)
12 Project.__init__(self, plotter_queue)
13
13
14 self.setDaemon(True)
14 self.setDaemon(True)
15
15
16 self.lock = threading.Lock()
16 self.lock = threading.Lock()
17 self.control = {'stop':False, 'pause':False}
17 self.control = { 'stop':False, 'pause':False }
18
18
19 def __del__(self):
19 def __del__(self):
20
20
21 self.control['stop'] = True
21 self.control['stop'] = True
22
22
23 def stop(self):
23 def stop(self):
24
24
25 self.lock.acquire()
25 self.lock.acquire()
26
26
27 self.control['stop'] = True
27 self.control['stop'] = True
28
28
29 self.lock.release()
29 self.lock.release()
30
30
31 def pause(self):
31 def pause(self):
32
32
33 self.lock.acquire()
33 self.lock.acquire()
34
34
35 self.control['pause'] = not(self.control['pause'])
35 self.control['pause'] = not(self.control['pause'])
36 paused = self.control['pause']
36 paused = self.control['pause']
37
37
38 self.lock.release()
38 self.lock.release()
39
39
40 return paused
40 return paused
41
41
42 def isPaused(self):
42 def isPaused(self):
43
43
44 self.lock.acquire()
44 self.lock.acquire()
45 paused = self.control['pause']
45 paused = self.control['pause']
46 self.lock.release()
46 self.lock.release()
47
47
48 return paused
48 return paused
49
49
50 def isStopped(self):
50 def isStopped(self):
51
51
52 self.lock.acquire()
52 self.lock.acquire()
53 stopped = self.control['stop']
53 stopped = self.control['stop']
54 self.lock.release()
54 self.lock.release()
55
55
56 return stopped
56 return stopped
57
57
58 def run(self):
58 def run(self):
59 self.control['stop'] = False
59 self.control['stop'] = False
60 self.control['pause'] = False
60 self.control['pause'] = False
61
61
62 self.writeXml()
62 self.writeXml()
63
63
64 self.createObjects()
64 self.createObjects()
65 self.connectObjects()
65 self.connectObjects()
66 Project.run(self)
66 Project.run(self)
67
67
68 def isRunning(self):
68 def isRunning(self):
69
69
70 return self.is_alive()
70 return self.is_alive()
71
71
72 def isFinished(self):
72 def isFinished(self):
73
73
74 return not self.is_alive()
74 return not self.is_alive()
75
75
76 def setPlotters(self):
76 def setPlotters(self):
77
77
78 plotterList = PlotManager.plotterList
78 plotterList = PlotManager.plotterList
79
79
80 for thisPUConfObj in self.procUnitConfObjDict.values():
80 for thisPUConfObj in self.procUnitConfObjDict.values():
81
81
82 inputId = thisPUConfObj.getInputId()
82 inputId = thisPUConfObj.getInputId()
83
83
84 if int(inputId) == 0:
84 if int(inputId) == 0:
85 continue
85 continue
86
86
87 for thisOpObj in thisPUConfObj.getOperationObjList():
87 for thisOpObj in thisPUConfObj.getOperationObjList():
88
88
89 if thisOpObj.type == "self":
89 if thisOpObj.type == "self":
90 continue
90 continue
91
91
92 if thisOpObj.name in plotterList:
92 if thisOpObj.name in plotterList:
93 thisOpObj.type = "plotter"
93 thisOpObj.type = "other"
94
94
95 def setPlotterQueue(self, plotter_queue):
95 def setPlotterQueue(self, plotter_queue):
96
96
97 self.plotterQueue = plotter_queue
97 self.plotterQueue = plotter_queue
98
98
99 def getPlotterQueue(self):
99 def getPlotterQueue(self):
100
100
101 return self.plotterQueue
101 return self.plotterQueue
102
102
103 def useExternalPlotter(self):
103 def useExternalPlotter(self):
104
104
105 self.plotterQueue = Queue(10)
105 self.plotterQueue = Queue(10)
106 self.setPlotters()
106 self.setPlotters()
107
107
108 plotManagerObj = PlotManager(self.plotterQueue)
108 plotManagerObj = PlotManager(self.plotterQueue)
109 plotManagerObj.setController(self)
109 plotManagerObj.setController(self)
110
110
111 return plotManagerObj
111 return plotManagerObj
112
112
113 # from PyQt4 import QtCore
113 # from PyQt4 import QtCore
114 # from PyQt4.QtCore import SIGNAL
114 # from PyQt4.QtCore import SIGNAL
115 #
115 #
116 # class ControllerQThread(QtCore.QThread, Project):
116 # class ControllerQThread(QtCore.QThread, Project):
117 #
117 #
118 # def __init__(self, filename):
118 # def __init__(self, filename):
119 #
119 #
120 # QtCore.QThread.__init__(self)
120 # QtCore.QThread.__init__(self)
121 # Project.__init__(self)
121 # Project.__init__(self)
122 #
122 #
123 # self.filename = filename
123 # self.filename = filename
124 #
124 #
125 # self.lock = threading.Lock()
125 # self.lock = threading.Lock()
126 # self.control = {'stop':False, 'pause':False}
126 # self.control = {'stop':False, 'pause':False}
127 #
127 #
128 # def __del__(self):
128 # def __del__(self):
129 #
129 #
130 # self.control['stop'] = True
130 # self.control['stop'] = True
131 # self.wait()
131 # self.wait()
132 #
132 #
133 # def stop(self):
133 # def stop(self):
134 #
134 #
135 # self.lock.acquire()
135 # self.lock.acquire()
136 #
136 #
137 # self.control['stop'] = True
137 # self.control['stop'] = True
138 #
138 #
139 # self.lock.release()
139 # self.lock.release()
140 #
140 #
141 # def pause(self):
141 # def pause(self):
142 #
142 #
143 # self.lock.acquire()
143 # self.lock.acquire()
144 #
144 #
145 # self.control['pause'] = not(self.control['pause'])
145 # self.control['pause'] = not(self.control['pause'])
146 # paused = self.control['pause']
146 # paused = self.control['pause']
147 #
147 #
148 # self.lock.release()
148 # self.lock.release()
149 #
149 #
150 # return paused
150 # return paused
151 #
151 #
152 # def isPaused(self):
152 # def isPaused(self):
153 #
153 #
154 # self.lock.acquire()
154 # self.lock.acquire()
155 # paused = self.control['pause']
155 # paused = self.control['pause']
156 # self.lock.release()
156 # self.lock.release()
157 #
157 #
158 # return paused
158 # return paused
159 #
159 #
160 # def isStopped(self):
160 # def isStopped(self):
161 #
161 #
162 # self.lock.acquire()
162 # self.lock.acquire()
163 # stopped = self.control['stop']
163 # stopped = self.control['stop']
164 # self.lock.release()
164 # self.lock.release()
165 #
165 #
166 # return stopped
166 # return stopped
167 #
167 #
168 # def run(self):
168 # def run(self):
169 #
169 #
170 # self.control['stop'] = False
170 # self.control['stop'] = False
171 # self.control['pause'] = False
171 # self.control['pause'] = False
172 #
172 #
173 # self.readXml(self.filename)
173 # self.readXml(self.filename)
174 # self.createObjects()
174 # self.createObjects()
175 # self.connectObjects()
175 # self.connectObjects()
176 # self.emit( SIGNAL( "jobStarted( PyQt_PyObject )" ), 1)
176 # self.emit( SIGNAL( "jobStarted( PyQt_PyObject )" ), 1)
177 # Project.run(self)
177 # Project.run(self)
178 # self.emit( SIGNAL( "jobFinished( PyQt_PyObject )" ), 1)
178 # self.emit( SIGNAL( "jobFinished( PyQt_PyObject )" ), 1)
179 #
179 #
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
Show file before | Show file after | Hide comments
@@ -1,1007 +1,1009
1 '''
1 '''
2 @author: Daniel Suarez
2 @author: Daniel Suarez
3 '''
3 '''
4 import os
4 import os
5 import glob
5 import glob
6 import ftplib
6 import ftplib
7
7
8 try:
8 try:
9 import paramiko
9 import paramiko
10 import scp
10 import scp
11 except:
11 except:
12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
13
13
14 import time
14 import time
15
15
16 import threading
16 import threading
17 Thread = threading.Thread
17 Thread = threading.Thread
18
18
19 # try:
19 # try:
20 # from gevent import sleep
20 # from gevent import sleep
21 # except:
21 # except:
22 from time import sleep
22 from time import sleep
23
23
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25
25
26 class Remote(Thread):
26 class Remote(Thread):
27 """
27 """
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 used to upload or download files remotely.
29 used to upload or download files remotely.
30
30
31 Non-standard Python modules used:
31 Non-standard Python modules used:
32 None
32 None
33
33
34 Written by:
34 Written by:
35
35
36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
37
37
38 """
38 """
39
39
40 server = None
40 server = None
41 username = None
41 username = None
42 password = None
42 password = None
43 remotefolder = None
43 remotefolder = None
44
44
45 period = 60
45 period = 60
46 fileList = []
46 fileList = []
47 bussy = False
47 bussy = False
48
48
49 def __init__(self, server, username, password, remotefolder, period=60):
49 def __init__(self, server, username, password, remotefolder, period=60):
50
50
51 Thread.__init__(self)
51 Thread.__init__(self)
52
52
53 self.setDaemon(True)
53 self.setDaemon(True)
54
54
55 self.status = 0
55 self.status = 0
56
56
57 self.__server = server
57 self.__server = server
58 self.__username = username
58 self.__username = username
59 self.__password = password
59 self.__password = password
60 self.__remotefolder = remotefolder
60 self.__remotefolder = remotefolder
61
61
62 self.period = period
62 self.period = period
63
63
64 self.fileList = []
64 self.fileList = []
65 self.bussy = False
65 self.bussy = False
66
66
67 self.stopFlag = False
67 self.stopFlag = False
68
68
69 print "[Remote Server] Opening server: %s" %self.__server
69 print "[Remote Server] Opening server: %s" %self.__server
70 if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
70 if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
71 print "[Remote Server] %s server was opened successfully" %self.__server
71 print "[Remote Server] %s server was opened successfully" %self.__server
72
72
73 self.close()
73 self.close()
74
74
75 self.mutex = threading.Lock()
75 self.mutex = threading.Lock()
76
76
77 def stop(self):
77 def stop(self):
78
78
79 self.stopFlag = True
79 self.stopFlag = True
80 self.join(10)
80 self.join(10)
81
81
82 def open(self):
82 def open(self):
83 """
83 """
84 Connect to server and create a connection class (FTP or SSH) to remote server.
84 Connect to server and create a connection class (FTP or SSH) to remote server.
85 """
85 """
86 raise NotImplementedError, "Implement this method in child class"
86 raise NotImplementedError, "Implement this method in child class"
87
87
88 def close(self):
88 def close(self):
89 """
89 """
90 Close connection to server
90 Close connection to server
91 """
91 """
92 raise NotImplementedError, "Implement this method in child class"
92 raise NotImplementedError, "Implement this method in child class"
93
93
94 def mkdir(self, remotefolder):
94 def mkdir(self, remotefolder):
95 """
95 """
96 Create a folder remotely
96 Create a folder remotely
97 """
97 """
98 raise NotImplementedError, "Implement this method in child class"
98 raise NotImplementedError, "Implement this method in child class"
99
99
100 def cd(self, remotefolder):
100 def cd(self, remotefolder):
101 """
101 """
102 Change working directory in remote server
102 Change working directory in remote server
103 """
103 """
104 raise NotImplementedError, "Implement this method in child class"
104 raise NotImplementedError, "Implement this method in child class"
105
105
106 def download(self, filename, localfolder=None):
106 def download(self, filename, localfolder=None):
107 """
107 """
108 Download a file from server to local host
108 Download a file from server to local host
109 """
109 """
110 raise NotImplementedError, "Implement this method in child class"
110 raise NotImplementedError, "Implement this method in child class"
111
111
112 def sendFile(self, fullfilename):
112 def sendFile(self, fullfilename):
113 """
113 """
114 sendFile method is used to upload a local file to the current directory in remote server
114 sendFile method is used to upload a local file to the current directory in remote server
115
115
116 Inputs:
116 Inputs:
117 fullfilename - full path name of local file to store in remote directory
117 fullfilename - full path name of local file to store in remote directory
118
118
119 Returns:
119 Returns:
120 0 in error case else 1
120 0 in error case else 1
121 """
121 """
122 raise NotImplementedError, "Implement this method in child class"
122 raise NotImplementedError, "Implement this method in child class"
123
123
124 def upload(self, fullfilename, remotefolder=None):
124 def upload(self, fullfilename, remotefolder=None):
125 """
125 """
126 upload method is used to upload a local file to remote directory. This method changes
126 upload method is used to upload a local file to remote directory. This method changes
127 working directory before sending a file.
127 working directory before sending a file.
128
128
129 Inputs:
129 Inputs:
130 fullfilename - full path name of local file to store in remote directory
130 fullfilename - full path name of local file to store in remote directory
131
131
132 remotefolder - remote directory
132 remotefolder - remote directory
133
133
134 Returns:
134 Returns:
135 0 in error case else 1
135 0 in error case else 1
136 """
136 """
137 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
137 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
138
138
139 if not self.status:
139 if not self.status:
140 return 0
140 return 0
141
141
142 if remotefolder == None:
142 if remotefolder == None:
143 remotefolder = self.remotefolder
143 remotefolder = self.remotefolder
144
144
145 if not self.cd(remotefolder):
145 if not self.cd(remotefolder):
146 return 0
146 return 0
147
147
148 if not self.sendFile(fullfilename):
148 if not self.sendFile(fullfilename):
149 print "[Remote Server] Error uploading file %s" %fullfilename
149 print "[Remote Server] Error uploading file %s" %fullfilename
150 return 0
150 return 0
151
151
152 print "[Remote Server] upload finished successfully"
152 print "[Remote Server] upload finished successfully"
153
153
154 return 1
154 return 1
155
155
156 def delete(self, filename):
156 def delete(self, filename):
157 """
157 """
158 Remove a file from remote server
158 Remove a file from remote server
159 """
159 """
160 pass
160 pass
161
161
162 def updateFileList(self, fileList):
162 def updateFileList(self, fileList):
163 """
163 """
164 Remove a file from remote server
164 Remove a file from remote server
165 """
165 """
166
166
167 if fileList == self.fileList:
167 if fileList == self.fileList:
168 return 0
168 return 0
169
169
170 self.mutex.acquire()
170 self.mutex.acquire()
171 # init = time.time()
171 # init = time.time()
172 #
172 #
173 # while(self.bussy):
173 # while(self.bussy):
174 # sleep(0.1)
174 # sleep(0.1)
175 # if time.time() - init > 2*self.period:
175 # if time.time() - init > 2*self.period:
176 # return 0
176 # return 0
177
177
178 self.fileList = fileList
178 self.fileList = fileList
179 self.mutex.release()
179 self.mutex.release()
180 return 1
180 return 1
181
181
182 def run(self):
182 def run(self):
183
183
184 if not self.status:
184 if not self.status:
185 print "Finishing FTP service"
185 print "Finishing FTP service"
186 return
186 return
187
187
188 if not self.cd(self.remotefolder):
188 if not self.cd(self.remotefolder):
189 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
189 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
190
190
191 while True:
191 while True:
192
192
193 for i in range(self.period):
193 for i in range(self.period):
194 if self.stopFlag:
194 if self.stopFlag:
195 break
195 break
196 sleep(1)
196 sleep(1)
197
197
198 if self.stopFlag:
198 if self.stopFlag:
199 break
199 break
200
200
201 # self.bussy = True
201 # self.bussy = True
202 self.mutex.acquire()
202 self.mutex.acquire()
203
203
204 print "[Remote Server] Opening %s" %self.__server
204 print "[Remote Server] Opening %s" %self.__server
205 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
205 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
206 self.mutex.release()
206 self.mutex.release()
207 continue
207 continue
208
208
209 for thisFile in self.fileList:
209 for thisFile in self.fileList:
210 self.upload(thisFile, self.remotefolder)
210 self.upload(thisFile, self.remotefolder)
211
211
212 print "[Remote Server] Closing %s" %self.__server
212 print "[Remote Server] Closing %s" %self.__server
213 self.close()
213 self.close()
214
214
215 self.mutex.release()
215 self.mutex.release()
216 # self.bussy = False
216 # self.bussy = False
217
217
218 print "[Remote Server] Thread stopped successfully"
218 print "[Remote Server] Thread stopped successfully"
219
219
220 class FTPClient(Remote):
220 class FTPClient(Remote):
221
221
222 __ftpClientObj = None
222 __ftpClientObj = None
223
223
224 def __init__(self, server, username, password, remotefolder, period=60):
224 def __init__(self, server, username, password, remotefolder, period=60):
225 """
225 """
226 """
226 """
227 Remote.__init__(self, server, username, password, remotefolder, period)
227 Remote.__init__(self, server, username, password, remotefolder, period)
228
228
229 def open(self, server, username, password, remotefolder):
229 def open(self, server, username, password, remotefolder):
230
230
231 """
231 """
232 This method is used to set FTP parameters and establish a connection to remote server
232 This method is used to set FTP parameters and establish a connection to remote server
233
233
234 Inputs:
234 Inputs:
235 server - remote server IP Address
235 server - remote server IP Address
236
236
237 username - remote server Username
237 username - remote server Username
238
238
239 password - remote server password
239 password - remote server password
240
240
241 remotefolder - remote server current working directory
241 remotefolder - remote server current working directory
242
242
243 Return:
243 Return:
244 Boolean - Returns 1 if a connection has been established, 0 otherwise
244 Boolean - Returns 1 if a connection has been established, 0 otherwise
245
245
246 Affects:
246 Affects:
247 self.status - in case of error or fail connection this parameter is set to 0 else 1
247 self.status - in case of error or fail connection this parameter is set to 0 else 1
248
248
249 """
249 """
250
250
251 if server == None:
251 if server == None:
252 raise ValueError, "FTP server should be defined"
252 raise ValueError, "FTP server should be defined"
253
253
254 if username == None:
254 if username == None:
255 raise ValueError, "FTP username should be defined"
255 raise ValueError, "FTP username should be defined"
256
256
257 if password == None:
257 if password == None:
258 raise ValueError, "FTP password should be defined"
258 raise ValueError, "FTP password should be defined"
259
259
260 if remotefolder == None:
260 if remotefolder == None:
261 raise ValueError, "FTP remote folder should be defined"
261 raise ValueError, "FTP remote folder should be defined"
262
262
263 try:
263 try:
264 ftpClientObj = ftplib.FTP(server)
264 ftpClientObj = ftplib.FTP(server)
265 except ftplib.all_errors, e:
265 except ftplib.all_errors, e:
266 print "[FTP Server]: FTP server connection fail: %s" %server
266 print "[FTP Server]: FTP server connection fail: %s" %server
267 print "[FTP Server]:", e
267 print "[FTP Server]:", e
268 self.status = 0
268 self.status = 0
269 return 0
269 return 0
270
270
271 try:
271 try:
272 ftpClientObj.login(username, password)
272 ftpClientObj.login(username, password)
273 except ftplib.all_errors:
273 except ftplib.all_errors:
274 print "[FTP Server]: FTP username or password are incorrect"
274 print "[FTP Server]: FTP username or password are incorrect"
275 self.status = 0
275 self.status = 0
276 return 0
276 return 0
277
277
278 if remotefolder == None:
278 if remotefolder == None:
279 remotefolder = ftpClientObj.pwd()
279 remotefolder = ftpClientObj.pwd()
280 else:
280 else:
281 try:
281 try:
282 ftpClientObj.cwd(remotefolder)
282 ftpClientObj.cwd(remotefolder)
283 except ftplib.all_errors:
283 except ftplib.all_errors:
284 print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
284 print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
285 remotefolder = ftpClientObj.pwd()
285 remotefolder = ftpClientObj.pwd()
286
286
287 self.server = server
287 self.server = server
288 self.username = username
288 self.username = username
289 self.password = password
289 self.password = password
290 self.remotefolder = remotefolder
290 self.remotefolder = remotefolder
291 self.__ftpClientObj = ftpClientObj
291 self.__ftpClientObj = ftpClientObj
292 self.status = 1
292 self.status = 1
293
293
294 return 1
294 return 1
295
295
296 def close(self):
296 def close(self):
297 """
297 """
298 Close connection to remote server
298 Close connection to remote server
299 """
299 """
300 if not self.status:
300 if not self.status:
301 return 0
301 return 0
302
302
303 self.__ftpClientObj.close()
303 self.__ftpClientObj.close()
304
304
305 def mkdir(self, remotefolder):
305 def mkdir(self, remotefolder):
306 """
306 """
307 mkdir is used to make a new directory in remote server
307 mkdir is used to make a new directory in remote server
308
308
309 Input:
309 Input:
310 remotefolder - directory name
310 remotefolder - directory name
311
311
312 Return:
312 Return:
313 0 in error case else 1
313 0 in error case else 1
314 """
314 """
315 if not self.status:
315 if not self.status:
316 return 0
316 return 0
317
317
318 try:
318 try:
319 self.__ftpClientObj.mkd(dirname)
319 self.__ftpClientObj.mkd(dirname)
320 except ftplib.all_errors:
320 except ftplib.all_errors:
321 print "[FTP Server]: Error creating remote folder: %s" %remotefolder
321 print "[FTP Server]: Error creating remote folder: %s" %remotefolder
322 return 0
322 return 0
323
323
324 return 1
324 return 1
325
325
326 def cd(self, remotefolder):
326 def cd(self, remotefolder):
327 """
327 """
328 cd is used to change remote working directory on server
328 cd is used to change remote working directory on server
329
329
330 Input:
330 Input:
331 remotefolder - current working directory
331 remotefolder - current working directory
332
332
333 Affects:
333 Affects:
334 self.remotefolder
334 self.remotefolder
335
335
336 Return:
336 Return:
337 0 in case of error else 1
337 0 in case of error else 1
338 """
338 """
339 if not self.status:
339 if not self.status:
340 return 0
340 return 0
341
341
342 if remotefolder == self.remotefolder:
342 if remotefolder == self.remotefolder:
343 return 1
343 return 1
344
344
345 try:
345 try:
346 self.__ftpClientObj.cwd(remotefolder)
346 self.__ftpClientObj.cwd(remotefolder)
347 except ftplib.all_errors:
347 except ftplib.all_errors:
348 print '[FTP Server]: Error changing to %s' %remotefolder
348 print '[FTP Server]: Error changing to %s' %remotefolder
349 print '[FTP Server]: Trying to create remote folder'
349 print '[FTP Server]: Trying to create remote folder'
350
350
351 if not self.mkdir(remotefolder):
351 if not self.mkdir(remotefolder):
352 print '[FTP Server]: Remote folder could not be created'
352 print '[FTP Server]: Remote folder could not be created'
353 return 0
353 return 0
354
354
355 try:
355 try:
356 self.__ftpClientObj.cwd(remotefolder)
356 self.__ftpClientObj.cwd(remotefolder)
357 except ftplib.all_errors:
357 except ftplib.all_errors:
358 return 0
358 return 0
359
359
360 self.remotefolder = remotefolder
360 self.remotefolder = remotefolder
361
361
362 return 1
362 return 1
363
363
364 def sendFile(self, fullfilename):
364 def sendFile(self, fullfilename):
365
365
366 if not self.status:
366 if not self.status:
367 return 0
367 return 0
368
368
369 fp = open(fullfilename, 'rb')
369 fp = open(fullfilename, 'rb')
370
370
371 filename = os.path.basename(fullfilename)
371 filename = os.path.basename(fullfilename)
372
372
373 command = "STOR %s" %filename
373 command = "STOR %s" %filename
374
374
375 try:
375 try:
376 self.__ftpClientObj.storbinary(command, fp)
376 self.__ftpClientObj.storbinary(command, fp)
377 except ftplib.all_errors, e:
377 except ftplib.all_errors, e:
378 print "[FTP Server]:", e
378 print "[FTP Server]:", e
379 return 0
379 return 0
380
380
381 try:
381 try:
382 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
382 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
383 except ftplib.all_errors, e:
383 except ftplib.all_errors, e:
384 print "[FTP Server]:", e
384 print "[FTP Server]:", e
385
385
386 fp.close()
386 fp.close()
387
387
388 return 1
388 return 1
389
389
390 class SSHClient(Remote):
390 class SSHClient(Remote):
391
391
392 __sshClientObj = None
392 __sshClientObj = None
393 __scpClientObj = None
393 __scpClientObj = None
394
394
395 def __init__(self, server, username, password, remotefolder, period=60):
395 def __init__(self, server, username, password, remotefolder, period=60):
396 """
396 """
397 """
397 """
398 Remote.__init__(self, server, username, password, remotefolder, period)
398 Remote.__init__(self, server, username, password, remotefolder, period)
399
399
400 def open(self, server, username, password, remotefolder, port=22):
400 def open(self, server, username, password, remotefolder, port=22):
401
401
402 """
402 """
403 This method is used to set SSH parameters and establish a connection to a remote server
403 This method is used to set SSH parameters and establish a connection to a remote server
404
404
405 Inputs:
405 Inputs:
406 server - remote server IP Address
406 server - remote server IP Address
407
407
408 username - remote server Username
408 username - remote server Username
409
409
410 password - remote server password
410 password - remote server password
411
411
412 remotefolder - remote server current working directory
412 remotefolder - remote server current working directory
413
413
414 Return: void
414 Return: void
415
415
416 Affects:
416 Affects:
417 self.status - in case of error or fail connection this parameter is set to 0 else 1
417 self.status - in case of error or fail connection this parameter is set to 0 else 1
418
418
419 """
419 """
420 import socket
420 import socket
421
421
422 if server == None:
422 if server == None:
423 raise ValueError, "SSH server should be defined"
423 raise ValueError, "SSH server should be defined"
424
424
425 if username == None:
425 if username == None:
426 raise ValueError, "SSH username should be defined"
426 raise ValueError, "SSH username should be defined"
427
427
428 if password == None:
428 if password == None:
429 raise ValueError, "SSH password should be defined"
429 raise ValueError, "SSH password should be defined"
430
430
431 if remotefolder == None:
431 if remotefolder == None:
432 raise ValueError, "SSH remote folder should be defined"
432 raise ValueError, "SSH remote folder should be defined"
433
433
434 sshClientObj = paramiko.SSHClient()
434 sshClientObj = paramiko.SSHClient()
435
435
436 sshClientObj.load_system_host_keys()
436 sshClientObj.load_system_host_keys()
437 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
437 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
438
438
439 self.status = 0
439 self.status = 0
440 try:
440 try:
441 sshClientObj.connect(server, username=username, password=password, port=port)
441 sshClientObj.connect(server, username=username, password=password, port=port)
442 except paramiko.AuthenticationException, e:
442 except paramiko.AuthenticationException, e:
443 # print "SSH username or password are incorrect: %s"
443 # print "SSH username or password are incorrect: %s"
444 print "[SSH Server]:", e
444 print "[SSH Server]:", e
445 return 0
445 return 0
446 except SSHException, e:
446 except SSHException, e:
447 print "[SSH Server]:", e
447 print "[SSH Server]:", e
448 return 0
448 return 0
449 except socket.error:
449 except socket.error:
450 self.status = 0
450 self.status = 0
451 print "[SSH Server]:", e
451 print "[SSH Server]:", e
452 return 0
452 return 0
453
453
454 self.status = 1
454 self.status = 1
455 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
455 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
456
456
457 if remotefolder == None:
457 if remotefolder == None:
458 remotefolder = self.pwd()
458 remotefolder = self.pwd()
459
459
460 self.server = server
460 self.server = server
461 self.username = username
461 self.username = username
462 self.password = password
462 self.password = password
463 self.__sshClientObj = sshClientObj
463 self.__sshClientObj = sshClientObj
464 self.__scpClientObj = scpClientObj
464 self.__scpClientObj = scpClientObj
465 self.status = 1
465 self.status = 1
466
466
467 if not self.cd(remotefolder):
467 if not self.cd(remotefolder):
468 raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
468 raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
469 return 0
469 return 0
470
470
471 self.remotefolder = remotefolder
471 self.remotefolder = remotefolder
472
472
473 return 1
473 return 1
474
474
475 def close(self):
475 def close(self):
476 """
476 """
477 Close connection to remote server
477 Close connection to remote server
478 """
478 """
479 if not self.status:
479 if not self.status:
480 return 0
480 return 0
481
481
482 self.__scpClientObj.close()
482 self.__scpClientObj.close()
483 self.__sshClientObj.close()
483 self.__sshClientObj.close()
484
484
485 def __execute(self, command):
485 def __execute(self, command):
486 """
486 """
487 __execute a command on remote server
487 __execute a command on remote server
488
488
489 Input:
489 Input:
490 command - Exmaple 'ls -l'
490 command - Exmaple 'ls -l'
491
491
492 Return:
492 Return:
493 0 in error case else 1
493 0 in error case else 1
494 """
494 """
495 if not self.status:
495 if not self.status:
496 return 0
496 return 0
497
497
498 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
498 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
499
499
500 result = stderr.readlines()
500 result = stderr.readlines()
501 if len(result) > 1:
501 if len(result) > 1:
502 return 0
502 return 0
503
503
504 result = stdout.readlines()
504 result = stdout.readlines()
505 if len(result) > 1:
505 if len(result) > 1:
506 return result[0][:-1]
506 return result[0][:-1]
507
507
508 return 1
508 return 1
509
509
510 def mkdir(self, remotefolder):
510 def mkdir(self, remotefolder):
511 """
511 """
512 mkdir is used to make a new directory in remote server
512 mkdir is used to make a new directory in remote server
513
513
514 Input:
514 Input:
515 remotefolder - directory name
515 remotefolder - directory name
516
516
517 Return:
517 Return:
518 0 in error case else 1
518 0 in error case else 1
519 """
519 """
520
520
521 command = 'mkdir %s' %remotefolder
521 command = 'mkdir %s' %remotefolder
522
522
523 return self.__execute(command)
523 return self.__execute(command)
524
524
525 def pwd(self):
525 def pwd(self):
526
526
527 command = 'pwd'
527 command = 'pwd'
528
528
529 return self.__execute(command)
529 return self.__execute(command)
530
530
531 def cd(self, remotefolder):
531 def cd(self, remotefolder):
532 """
532 """
533 cd is used to change remote working directory on server
533 cd is used to change remote working directory on server
534
534
535 Input:
535 Input:
536 remotefolder - current working directory
536 remotefolder - current working directory
537
537
538 Affects:
538 Affects:
539 self.remotefolder
539 self.remotefolder
540
540
541 Return:
541 Return:
542 0 in case of error else 1
542 0 in case of error else 1
543 """
543 """
544 if not self.status:
544 if not self.status:
545 return 0
545 return 0
546
546
547 if remotefolder == self.remotefolder:
547 if remotefolder == self.remotefolder:
548 return 1
548 return 1
549
549
550 chk_command = "cd %s; pwd" %remotefolder
550 chk_command = "cd %s; pwd" %remotefolder
551 mkdir_command = "mkdir %s" %remotefolder
551 mkdir_command = "mkdir %s" %remotefolder
552
552
553 if not self.__execute(chk_command):
553 if not self.__execute(chk_command):
554 if not self.__execute(mkdir_command):
554 if not self.__execute(mkdir_command):
555 self.remotefolder = None
555 self.remotefolder = None
556 return 0
556 return 0
557
557
558 self.remotefolder = remotefolder
558 self.remotefolder = remotefolder
559
559
560 return 1
560 return 1
561
561
562 def sendFile(self, fullfilename):
562 def sendFile(self, fullfilename):
563
563
564 if not self.status:
564 if not self.status:
565 return 0
565 return 0
566
566
567 try:
567 try:
568 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
568 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
569 except scp.ScpError, e:
569 except scp.ScpError, e:
570 print "[SSH Server]", str(e)
570 print "[SSH Server]", str(e)
571 return 0
571 return 0
572
572
573 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
573 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
574 command = 'chmod 775 %s' %remotefile
574 command = 'chmod 775 %s' %remotefile
575
575
576 return self.__execute(command)
576 return self.__execute(command)
577
577
578 class SendToServer(ProcessingUnit):
578 class SendToServer(ProcessingUnit):
579
579
580 def __init__(self, **kwargs):
580 def __init__(self, **kwargs):
581
581
582 ProcessingUnit.__init__(self, **kwargs)
582 ProcessingUnit.__init__(self, **kwargs)
583
583
584 self.isConfig = False
584 self.isConfig = False
585 self.clientObj = None
585 self.clientObj = None
586
586
587 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
587 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
588
588
589 self.clientObj = None
589 self.clientObj = None
590 self.localfolder = localfolder
590 self.localfolder = localfolder
591 self.ext = ext
591 self.ext = ext
592 self.period = period
592 self.period = period
593
593
594 if str.lower(protocol) == 'ftp':
594 if str.lower(protocol) == 'ftp':
595 self.clientObj = FTPClient(server, username, password, remotefolder, period)
595 self.clientObj = FTPClient(server, username, password, remotefolder, period)
596
596
597 if str.lower(protocol) == 'ssh':
597 if str.lower(protocol) == 'ssh':
598 self.clientObj = SSHClient(server, username, password, remotefolder, period)
598 self.clientObj = SSHClient(server, username, password, remotefolder, period)
599
599
600 if not self.clientObj:
600 if not self.clientObj:
601 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
601 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
602
602
603 self.clientObj.start()
603 self.clientObj.start()
604
604
605 def findFiles(self):
605 def findFiles(self):
606
606
607 if not type(self.localfolder) == list:
607 if not type(self.localfolder) == list:
608 folderList = [self.localfolder]
608 folderList = [self.localfolder]
609 else:
609 else:
610 folderList = self.localfolder
610 folderList = self.localfolder
611
611
612 #Remove duplicate items
612 #Remove duplicate items
613 folderList = list(set(folderList))
613 folderList = list(set(folderList))
614
614
615 fullfilenameList = []
615 fullfilenameList = []
616
616
617 for thisFolder in folderList:
617 for thisFolder in folderList:
618
618
619 print "[Remote Server]: Searching files on %s" %thisFolder
619 print "[Remote Server]: Searching files on %s" %thisFolder
620
620
621 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
621 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
622
622
623 if len(filenameList) < 1:
623 if len(filenameList) < 1:
624 continue
624 continue
625
625
626 for thisFile in filenameList:
626 for thisFile in filenameList:
627 fullfilename = os.path.join(thisFolder, thisFile)
627 fullfilename = os.path.join(thisFolder, thisFile)
628
628
629 if fullfilename in fullfilenameList:
629 if fullfilename in fullfilenameList:
630 continue
630 continue
631
631
632 #Only files modified in the last 30 minutes are considered
632 #Only files modified in the last 30 minutes are considered
633 if os.path.getmtime(fullfilename) < time.time() - 30*60:
633 if os.path.getmtime(fullfilename) < time.time() - 30*60:
634 continue
634 continue
635
635
636 fullfilenameList.append(fullfilename)
636 fullfilenameList.append(fullfilename)
637
637
638 return fullfilenameList
638 return fullfilenameList
639
639
640 def run(self, **kwargs):
640 def run(self, **kwargs):
641
642 if not self.isConfig:
641 if not self.isConfig:
643 self.init = time.time()
642 self.init = time.time()
644 self.setup(**kwargs)
643 self.setup(**kwargs)
645 self.isConfig = True
644 self.isConfig = True
646
645
646 if not self.clientObj.is_alive():
647 print "[Remote Server]: Restarting connection "
648 self.setup(**kwargs)
649
647 if time.time() - self.init >= self.period:
650 if time.time() - self.init >= self.period:
648 fullfilenameList = self.findFiles()
651 fullfilenameList = self.findFiles()
649
652
650 if self.clientObj.updateFileList(fullfilenameList):
653 if self.clientObj.updateFileList(fullfilenameList):
651 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
654 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
652
653 self.init = time.time()
655 self.init = time.time()
654
656
655 def close(self):
657 def close(self):
656 print "[Remote Server] Stopping thread"
658 print "[Remote Server] Stopping thread"
657 self.clientObj.stop()
659 self.clientObj.stop()
658
660
659
661
660 class FTP(object):
662 class FTP(object):
661 """
663 """
662 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
664 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
663
665
664 Non-standard Python modules used: None
666 Non-standard Python modules used: None
665
667
666 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
668 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
667 """
669 """
668
670
669 def __init__(self,server = None, username=None, password=None, remotefolder=None):
671 def __init__(self,server = None, username=None, password=None, remotefolder=None):
670 """
672 """
671 This method is used to setting parameters for FTP and establishing connection to remote server
673 This method is used to setting parameters for FTP and establishing connection to remote server
672
674
673 Inputs:
675 Inputs:
674 server - remote server IP Address
676 server - remote server IP Address
675
677
676 username - remote server Username
678 username - remote server Username
677
679
678 password - remote server password
680 password - remote server password
679
681
680 remotefolder - remote server current working directory
682 remotefolder - remote server current working directory
681
683
682 Return: void
684 Return: void
683
685
684 Affects:
686 Affects:
685 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
687 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
686
688
687 self.folderList - sub-folder list of remote folder
689 self.folderList - sub-folder list of remote folder
688
690
689 self.fileList - file list of remote folder
691 self.fileList - file list of remote folder
690
692
691
693
692 """
694 """
693
695
694 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
696 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
695 server, username, password, remotefolder = self.parmsByDefault()
697 server, username, password, remotefolder = self.parmsByDefault()
696
698
697 self.server = server
699 self.server = server
698 self.username = username
700 self.username = username
699 self.password = password
701 self.password = password
700 self.remotefolder = remotefolder
702 self.remotefolder = remotefolder
701 self.file = None
703 self.file = None
702 self.ftp = None
704 self.ftp = None
703 self.status = 0
705 self.status = 0
704
706
705 try:
707 try:
706 self.ftp = ftplib.FTP(self.server)
708 self.ftp = ftplib.FTP(self.server)
707 self.ftp.login(self.username,self.password)
709 self.ftp.login(self.username,self.password)
708 self.ftp.cwd(self.remotefolder)
710 self.ftp.cwd(self.remotefolder)
709 # print 'Connect to FTP Server: Successfully'
711 # print 'Connect to FTP Server: Successfully'
710
712
711 except ftplib.all_errors:
713 except ftplib.all_errors:
712 print 'Error FTP Service'
714 print 'Error FTP Service'
713 self.status = 1
715 self.status = 1
714 return
716 return
715
717
716
718
717
719
718 self.dirList = []
720 self.dirList = []
719
721
720 try:
722 try:
721 self.dirList = self.ftp.nlst()
723 self.dirList = self.ftp.nlst()
722
724
723 except ftplib.error_perm, resp:
725 except ftplib.error_perm, resp:
724 if str(resp) == "550 No files found":
726 if str(resp) == "550 No files found":
725 print "no files in this directory"
727 print "no files in this directory"
726 self.status = 1
728 self.status = 1
727 return
729 return
728
730
729 except ftplib.all_errors:
731 except ftplib.all_errors:
730 print 'Error Displaying Dir-Files'
732 print 'Error Displaying Dir-Files'
731 self.status = 1
733 self.status = 1
732 return
734 return
733
735
734 self.fileList = []
736 self.fileList = []
735 self.folderList = []
737 self.folderList = []
736 #only for test
738 #only for test
737 for f in self.dirList:
739 for f in self.dirList:
738 name, ext = os.path.splitext(f)
740 name, ext = os.path.splitext(f)
739 if ext != '':
741 if ext != '':
740 self.fileList.append(f)
742 self.fileList.append(f)
741 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
743 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
742
744
743 def parmsByDefault(self):
745 def parmsByDefault(self):
744 server = 'jro-app.igp.gob.pe'
746 server = 'jro-app.igp.gob.pe'
745 username = 'wmaster'
747 username = 'wmaster'
746 password = 'mst2010vhf'
748 password = 'mst2010vhf'
747 remotefolder = '/home/wmaster/graficos'
749 remotefolder = '/home/wmaster/graficos'
748
750
749 return server, username, password, remotefolder
751 return server, username, password, remotefolder
750
752
751
753
752 def mkd(self,dirname):
754 def mkd(self,dirname):
753 """
755 """
754 mkd is used to make directory in remote server
756 mkd is used to make directory in remote server
755
757
756 Input:
758 Input:
757 dirname - directory name
759 dirname - directory name
758
760
759 Return:
761 Return:
760 1 in error case else 0
762 1 in error case else 0
761 """
763 """
762 try:
764 try:
763 self.ftp.mkd(dirname)
765 self.ftp.mkd(dirname)
764 except:
766 except:
765 print 'Error creating remote folder:%s'%dirname
767 print 'Error creating remote folder:%s'%dirname
766 return 1
768 return 1
767
769
768 return 0
770 return 0
769
771
770
772
771 def delete(self,filename):
773 def delete(self,filename):
772 """
774 """
773 delete is used to delete file in current working directory of remote server
775 delete is used to delete file in current working directory of remote server
774
776
775 Input:
777 Input:
776 filename - filename to delete in remote folder
778 filename - filename to delete in remote folder
777
779
778 Return:
780 Return:
779 1 in error case else 0
781 1 in error case else 0
780 """
782 """
781
783
782 try:
784 try:
783 self.ftp.delete(filename)
785 self.ftp.delete(filename)
784 except:
786 except:
785 print 'Error deleting remote file:%s'%filename
787 print 'Error deleting remote file:%s'%filename
786 return 1
788 return 1
787
789
788 return 0
790 return 0
789
791
790 def download(self,filename,localfolder):
792 def download(self,filename,localfolder):
791 """
793 """
792 download is used to downloading file from remote folder into local folder
794 download is used to downloading file from remote folder into local folder
793
795
794 Inputs:
796 Inputs:
795 filename - filename to donwload
797 filename - filename to donwload
796
798
797 localfolder - directory local to store filename
799 localfolder - directory local to store filename
798
800
799 Returns:
801 Returns:
800 self.status - 1 in error case else 0
802 self.status - 1 in error case else 0
801 """
803 """
802
804
803 self.status = 0
805 self.status = 0
804
806
805
807
806 if not(filename in self.fileList):
808 if not(filename in self.fileList):
807 print 'filename:%s not exists'%filename
809 print 'filename:%s not exists'%filename
808 self.status = 1
810 self.status = 1
809 return self.status
811 return self.status
810
812
811 newfilename = os.path.join(localfolder,filename)
813 newfilename = os.path.join(localfolder,filename)
812
814
813 self.file = open(newfilename, 'wb')
815 self.file = open(newfilename, 'wb')
814
816
815 try:
817 try:
816 print 'Download: ' + filename
818 print 'Download: ' + filename
817 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
819 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
818 print 'Download Complete'
820 print 'Download Complete'
819 except ftplib.all_errors:
821 except ftplib.all_errors:
820 print 'Error Downloading ' + filename
822 print 'Error Downloading ' + filename
821 self.status = 1
823 self.status = 1
822 return self.status
824 return self.status
823
825
824 self.file.close()
826 self.file.close()
825
827
826 return self.status
828 return self.status
827
829
828
830
829 def __handleDownload(self,block):
831 def __handleDownload(self,block):
830 """
832 """
831 __handleDownload is used to handle writing file
833 __handleDownload is used to handle writing file
832 """
834 """
833 self.file.write(block)
835 self.file.write(block)
834
836
835
837
836 def upload(self,filename,remotefolder=None):
838 def upload(self,filename,remotefolder=None):
837 """
839 """
838 upload is used to uploading local file to remote directory
840 upload is used to uploading local file to remote directory
839
841
840 Inputs:
842 Inputs:
841 filename - full path name of local file to store in remote directory
843 filename - full path name of local file to store in remote directory
842
844
843 remotefolder - remote directory
845 remotefolder - remote directory
844
846
845 Returns:
847 Returns:
846 self.status - 1 in error case else 0
848 self.status - 1 in error case else 0
847 """
849 """
848
850
849 if remotefolder == None:
851 if remotefolder == None:
850 remotefolder = self.remotefolder
852 remotefolder = self.remotefolder
851
853
852 self.status = 0
854 self.status = 0
853
855
854 try:
856 try:
855 self.ftp.cwd(remotefolder)
857 self.ftp.cwd(remotefolder)
856
858
857 self.file = open(filename, 'rb')
859 self.file = open(filename, 'rb')
858
860
859 (head, tail) = os.path.split(filename)
861 (head, tail) = os.path.split(filename)
860
862
861 command = "STOR " + tail
863 command = "STOR " + tail
862
864
863 print 'Uploading: ' + tail
865 print 'Uploading: ' + tail
864 self.ftp.storbinary(command, self.file)
866 self.ftp.storbinary(command, self.file)
865 print 'Upload Completed'
867 print 'Upload Completed'
866
868
867 except ftplib.all_errors:
869 except ftplib.all_errors:
868 print 'Error Uploading ' + tail
870 print 'Error Uploading ' + tail
869 self.status = 1
871 self.status = 1
870 return self.status
872 return self.status
871
873
872 self.file.close()
874 self.file.close()
873
875
874 #back to initial directory in __init__()
876 #back to initial directory in __init__()
875 self.ftp.cwd(self.remotefolder)
877 self.ftp.cwd(self.remotefolder)
876
878
877 return self.status
879 return self.status
878
880
879
881
880 def dir(self,remotefolder):
882 def dir(self,remotefolder):
881 """
883 """
882 dir is used to change working directory of remote server and get folder and file list
884 dir is used to change working directory of remote server and get folder and file list
883
885
884 Input:
886 Input:
885 remotefolder - current working directory
887 remotefolder - current working directory
886
888
887 Affects:
889 Affects:
888 self.fileList - file list of working directory
890 self.fileList - file list of working directory
889
891
890 Return:
892 Return:
891 infoList - list with filenames and size of file in bytes
893 infoList - list with filenames and size of file in bytes
892
894
893 self.folderList - folder list
895 self.folderList - folder list
894 """
896 """
895
897
896 self.remotefolder = remotefolder
898 self.remotefolder = remotefolder
897 print 'Change to ' + self.remotefolder
899 print 'Change to ' + self.remotefolder
898 try:
900 try:
899 self.ftp.cwd(remotefolder)
901 self.ftp.cwd(remotefolder)
900 except ftplib.all_errors:
902 except ftplib.all_errors:
901 print 'Error Change to ' + self.remotefolder
903 print 'Error Change to ' + self.remotefolder
902 infoList = None
904 infoList = None
903 self.folderList = None
905 self.folderList = None
904 return infoList,self.folderList
906 return infoList,self.folderList
905
907
906 self.dirList = []
908 self.dirList = []
907
909
908 try:
910 try:
909 self.dirList = self.ftp.nlst()
911 self.dirList = self.ftp.nlst()
910
912
911 except ftplib.error_perm, resp:
913 except ftplib.error_perm, resp:
912 if str(resp) == "550 No files found":
914 if str(resp) == "550 No files found":
913 print "no files in this directory"
915 print "no files in this directory"
914 infoList = None
916 infoList = None
915 self.folderList = None
917 self.folderList = None
916 return infoList,self.folderList
918 return infoList,self.folderList
917 except ftplib.all_errors:
919 except ftplib.all_errors:
918 print 'Error Displaying Dir-Files'
920 print 'Error Displaying Dir-Files'
919 infoList = None
921 infoList = None
920 self.folderList = None
922 self.folderList = None
921 return infoList,self.folderList
923 return infoList,self.folderList
922
924
923 infoList = []
925 infoList = []
924 self.fileList = []
926 self.fileList = []
925 self.folderList = []
927 self.folderList = []
926 for f in self.dirList:
928 for f in self.dirList:
927 name,ext = os.path.splitext(f)
929 name,ext = os.path.splitext(f)
928 if ext != '':
930 if ext != '':
929 self.fileList.append(f)
931 self.fileList.append(f)
930 value = (f,self.ftp.size(f))
932 value = (f,self.ftp.size(f))
931 infoList.append(value)
933 infoList.append(value)
932
934
933 if ext == '':
935 if ext == '':
934 self.folderList.append(f)
936 self.folderList.append(f)
935
937
936 return infoList,self.folderList
938 return infoList,self.folderList
937
939
938
940
939 def close(self):
941 def close(self):
940 """
942 """
941 close is used to close and end FTP connection
943 close is used to close and end FTP connection
942
944
943 Inputs: None
945 Inputs: None
944
946
945 Return: void
947 Return: void
946
948
947 """
949 """
948 self.ftp.close()
950 self.ftp.close()
949
951
950 class SendByFTP(Operation):
952 class SendByFTP(Operation):
951
953
952 def __init__(self, **kwargs):
954 def __init__(self, **kwargs):
953 Operation.__init__(self, **kwargs)
955 Operation.__init__(self, **kwargs)
954 self.status = 1
956 self.status = 1
955 self.counter = 0
957 self.counter = 0
956
958
957 def error_print(self, ValueError):
959 def error_print(self, ValueError):
958
960
959 print ValueError, 'Error FTP'
961 print ValueError, 'Error FTP'
960 print "don't worry the program is running..."
962 print "don't worry the program is running..."
961
963
962 def worker_ftp(self, server, username, password, remotefolder, filenameList):
964 def worker_ftp(self, server, username, password, remotefolder, filenameList):
963
965
964 self.ftpClientObj = FTP(server, username, password, remotefolder)
966 self.ftpClientObj = FTP(server, username, password, remotefolder)
965 for filename in filenameList:
967 for filename in filenameList:
966 self.ftpClientObj.upload(filename)
968 self.ftpClientObj.upload(filename)
967 self.ftpClientObj.close()
969 self.ftpClientObj.close()
968
970
969 def ftp_thread(self, server, username, password, remotefolder):
971 def ftp_thread(self, server, username, password, remotefolder):
970 if not(self.status):
972 if not(self.status):
971 return
973 return
972
974
973 import multiprocessing
975 import multiprocessing
974
976
975 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
977 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
976 p.start()
978 p.start()
977
979
978 p.join(3)
980 p.join(3)
979
981
980 if p.is_alive():
982 if p.is_alive():
981 p.terminate()
983 p.terminate()
982 p.join()
984 p.join()
983 print 'killing ftp process...'
985 print 'killing ftp process...'
984 self.status = 0
986 self.status = 0
985 return
987 return
986
988
987 self.status = 1
989 self.status = 1
988 return
990 return
989
991
990 def filterByExt(self, ext, localfolder):
992 def filterByExt(self, ext, localfolder):
991 fnameList = glob.glob1(localfolder,ext)
993 fnameList = glob.glob1(localfolder,ext)
992 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
994 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
993
995
994 if len(self.filenameList) == 0:
996 if len(self.filenameList) == 0:
995 self.status = 0
997 self.status = 0
996
998
997 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
999 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
998
1000
999 self.counter += 1
1001 self.counter += 1
1000 if self.counter >= period:
1002 if self.counter >= period:
1001 self.filterByExt(ext, localfolder)
1003 self.filterByExt(ext, localfolder)
1002
1004
1003 self.ftp_thread(server, username, password, remotefolder)
1005 self.ftp_thread(server, username, password, remotefolder)
1004
1006
1005 self.counter = 0
1007 self.counter = 0
1006
1008
1007 self.status = 1
1009 self.status = 1
Show file before | Show file after | Hide comments
@@ -1,113 +1,117
1 import h5py
1 import h5py
2 import numpy
2 import numpy
3 import matplotlib.pyplot as plt
3 import matplotlib.pyplot as plt
4 import glob
4 import glob
5 import os
5 import os
6
6
7 #---------------------- Functions ---------------------
7 #---------------------- Functions ---------------------
8
8
9 def findFiles(path):
9 def findFiles(path):
10
10
11 dirList = []
11 dirList = []
12 fileList = []
12 fileList = []
13
13
14 for thisPath in os.listdir(path):
14 for thisPath in os.listdir(path):
15 dirList.append(os.path.join(path,thisPath))
15 dirList.append(os.path.join(path,thisPath))
16 dirList.sort()
16 dirList.sort()
17
17
18 for thisDirectory in dirList:
18 for thisDirectory in dirList:
19 files = glob.glob1(thisDirectory, "*.hdf5")
19 files = glob.glob1(thisDirectory, "*.hdf5")
20 files.sort()
20 files.sort()
21 for thisFile in files:
21 for thisFile in files:
22 fileList.append(os.path.join(thisDirectory,thisFile))
22 fileList.append(os.path.join(thisDirectory,thisFile))
23
23
24 return fileList
24 return fileList
25
25
26 def readFiles(fileList):
26 def readFiles(fileList):
27
27
28 meteors_array = numpy.zeros((1,4))
28 meteors_array = numpy.zeros((1,4))
29
29
30 for thisFile in fileList:
30 for thisFile in fileList:
31
31
32 #Leer
32 #Leer
33 f1 = h5py.File(thisFile,'r')
33 f1 = h5py.File(thisFile,'r')
34 grp1 = f1['Data']
34 grp1 = f1['Data']
35 grp2 = grp1['data_output']
35 grp2 = grp1['data_output']
36 meteors1 = grp2['table0'][:]
36 meteors1 = grp2['table0'][:]
37 meteors_array = numpy.vstack((meteors_array,meteors1))
37 meteors_array = numpy.vstack((meteors_array,meteors1))
38 #cerrar
38 #cerrar
39 f1.close()
39 f1.close()
40
40
41 meteors_array = numpy.delete(meteors_array, 0, axis=0)
41 meteors_array = numpy.delete(meteors_array, 0, axis=0)
42 meteors_list = [meteors_array[:,0],meteors_array[:,1],meteors_array[:,2],meteors_array[:,3]]
42 meteors_list = [meteors_array[:,0],meteors_array[:,1],meteors_array[:,2],meteors_array[:,3]]
43 return meteors_list
43 return meteors_list
44
44
45 def estimateMean(offset_list):
45 def estimateMean(offset_list):
46
46
47 mean_off = []
47 mean_off = []
48 axisY_off = []
48 axisY_off = []
49 axisX_off = []
49 axisX_off = []
50
50
51 for thisOffset in offset_list:
51 for thisOffset in offset_list:
52 mean_aux = numpy.mean(thisOffset, axis = 0)
52 mean_aux = numpy.mean(thisOffset, axis = 0)
53 mean_off.append(mean_aux)
53 mean_off.append(mean_aux)
54 axisX_off.append(numpy.array([0,numpy.size(thisOffset)]))
54 axisX_off.append(numpy.array([0,numpy.size(thisOffset)]))
55 axisY_off.append(numpy.array([mean_aux,mean_aux]))
55 axisY_off.append(numpy.array([mean_aux,mean_aux]))
56
56
57 return mean_off, axisY_off, axisX_off
57 return mean_off, axisY_off, axisX_off
58
58
59 def plotPhases(offset0, axisY0, axisX0, title):
59 def plotPhases(offset0, axisY0, axisX0, title):
60 f, axarr = plt.subplots(4, sharey=True)
60 f, axarr = plt.subplots(4, sharey=True)
61 color = ['b','g','r','c']
61 color = ['b','g','r','c']
62 # plt.grid()
62 # plt.grid()
63 for i in range(len(offset0)):
63 for i in range(len(offset0)):
64 thisMeteor = offset0[i]
64 thisMeteor = offset0[i]
65 thisY = axisY0[i]
65 thisY = axisY0[i]
66 thisX = axisX0[i]
66 thisX = axisX0[i]
67 thisColor = color[i]
67 thisColor = color[i]
68
68
69 opt = thisColor + 'o'
69 opt = thisColor + 'o'
70 axarr[i].plot(thisMeteor,opt)
70 axarr[i].plot(thisMeteor,opt)
71 axarr[i].plot(thisX, thisY, thisColor)
71 axarr[i].plot(thisX, thisY, thisColor)
72 axarr[i].set_ylabel('Offset ' + str(i))
72 axarr[i].set_ylabel('Offset ' + str(i))
73
73
74 plt.ylim((-180,180))
74 plt.ylim((-180,180))
75 axarr[0].set_title(title + ' Offsets')
75 axarr[0].set_title(title + ' Offsets')
76 axarr[3].set_xlabel('Number of estimations')
76 axarr[3].set_xlabel('Number of estimations')
77
77
78 return
78 return
79
79
80 def filterOffsets(offsets0, stdvLimit):
80 def filterOffsets(offsets0, stdvLimit):
81 offsets1 = []
81 offsets1 = []
82
82
83 for thisOffset in offsets0:
83 for thisOffset in offsets0:
84 pstd = numpy.std(thisOffset)*stdvLimit
84 pstd = numpy.std(thisOffset)*stdvLimit
85 pmean = numpy.mean(thisOffset)
85 pmean = numpy.mean(thisOffset)
86 outlier1 = thisOffset > pmean - pstd
86 outlier1 = thisOffset > pmean - pstd
87 outlier2 = thisOffset < pmean + pstd
87 outlier2 = thisOffset < pmean + pstd
88 not_outlier = numpy.logical_and(outlier1,outlier2)
88 not_outlier = numpy.logical_and(outlier1,outlier2)
89 thisOffset1 = thisOffset[not_outlier]
89 thisOffset1 = thisOffset[not_outlier]
90 offsets1.append(thisOffset1)
90 offsets1.append(thisOffset1)
91
91
92 return offsets1
92 return offsets1
93
93
94 #---------------------- Setup ---------------------------
94 #---------------------- Setup ---------------------------
95
95
96 <<<<<<< HEAD
96 path = '/home/nanosat/Pictures/JASMET30_mp/201608/phase'
97 path = '/home/nanosat/Pictures/JASMET30_mp/201608/phase'
98 =======
99 path = '/home/jespinoza/Pictures/JASMET30/201608/phase'
100 >>>>>>> master
97 stdvLimit = 0.5
101 stdvLimit = 0.5
98
102
99 #---------------------- Script ---------------------------
103 #---------------------- Script ---------------------------
100
104
101 fileList = findFiles(path)
105 fileList = findFiles(path)
102 offsets0 = readFiles(fileList)
106 offsets0 = readFiles(fileList)
103 mean0, axisY0, axisX0 = estimateMean(offsets0)
107 mean0, axisY0, axisX0 = estimateMean(offsets0)
104 plotPhases(offsets0, axisY0, axisX0, 'Original')
108 plotPhases(offsets0, axisY0, axisX0, 'Original')
105
109
106 offsets1 = filterOffsets(offsets0, stdvLimit)
110 offsets1 = filterOffsets(offsets0, stdvLimit)
107 mean1, axisY1, axisX1 = estimateMean(offsets1)
111 mean1, axisY1, axisX1 = estimateMean(offsets1)
108 plotPhases(offsets1, axisY1, axisX1, 'Filtered')
112 plotPhases(offsets1, axisY1, axisX1, 'Filtered')
109
113
110 print "Original Offsets: %.2f, %.2f, %.2f, %.2f" % (mean0[0],mean0[1],mean0[2],mean0[3])
114 print "Original Offsets: %.2f, %.2f, %.2f, %.2f" % (mean0[0],mean0[1],mean0[2],mean0[3])
111 print "Filtered Offsets: %.2f, %.2f, %.2f, %.2f" % (mean1[0],mean1[1],mean1[2],mean1[3])
115 print "Filtered Offsets: %.2f, %.2f, %.2f, %.2f" % (mean1[0],mean1[1],mean1[2],mean1[3])
112
116
113 plt.show()
117 plt.show()
General Comments 0
You need to be logged in to leave comments. Login now