##// END OF EJS Templates
schain
RSS
this fixes bug #1074 and bug #1101
José Chávez -
r1011:6db081ca94fe
parent child
Show More
Side by Side Unified
Context file:
r1011:6db081ca94fe Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,69
1 {
2 "configurations": [
3 {
4 "name": "Mac",
5 "includePath": [
6 "/usr/include",
7 "/usr/local/include",
8 "${workspaceRoot}"
9 ],
10 "defines": [],
11 "intelliSenseMode": "clang-x64",
12 "browse": {
13 "path": [
14 "/usr/include",
15 "/usr/local/include",
16 "${workspaceRoot}"
17 ],
18 "limitSymbolsToIncludedHeaders": true,
19 "databaseFilename": ""
20 }
21 },
22 {
23 "name": "Linux",
24 "includePath": [
25 "/usr/include/c++/4.8.4",
26 "/usr/include/x86_64-linux-gnu/c++/4.8",
27 "/usr/local/include",
28 "/usr/include",
29 "/usr/include/x86_64-linux-gnu",
30 "${workspaceRoot}"
31 ],
32 "defines": [],
33 "intelliSenseMode": "clang-x64",
34 "browse": {
35 "path": [
36 "/usr/include/c++/4.8.4",
37 "/usr/include/x86_64-linux-gnu/c++/4.8",
38 "/usr/local/include",
39 "/usr/include",
40 "/usr/include/x86_64-linux-gnu",
41 "${workspaceRoot}"
42 ],
43 "limitSymbolsToIncludedHeaders": true,
44 "databaseFilename": ""
45 }
46 },
47 {
48 "name": "Win32",
49 "includePath": [
50 "C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/include",
51 "${workspaceRoot}"
52 ],
53 "defines": [
54 "_DEBUG",
55 "UNICODE"
56 ],
57 "intelliSenseMode": "msvc-x64",
58 "browse": {
59 "path": [
60 "C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/include/*",
61 "${workspaceRoot}"
62 ],
63 "limitSymbolsToIncludedHeaders": true,
64 "databaseFilename": ""
65 }
66 }
67 ],
68 "version": 2
69 } No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,7
1 {
2 "files.associations": {
3 "arrayobject.h": "c",
4 "ndarrayobject.h": "c",
5 "complex": "c"
6 }
7 } No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,13 +1,108
1 .project
2 .pydevproject
3 build/
4 *.pyc
1 # Byte-compiled / optimized / DLL files
2 __pycache__/
3 *.py[cod]
4 *$py.class
5
6 # C extensions
5 7 *.so
8
9 # Distribution / packaging
10 .Python
11 env/
12 build/
13 develop-eggs/
14 dist/
15 downloads/
16 eggs/
17 .eggs/
18 lib/
19 lib64/
20 parts/
21 sdist/
22 var/
23 wheels/
6 24 *.egg-info/
25 .installed.cfg
26 *.egg
27
28 # PyInstaller
29 # Usually these files are written by a python script from a template
30 # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 *.manifest
32 *.spec
33
34 # Installer logs
35 pip-log.txt
36 pip-delete-this-directory.txt
37
38 # Unit test / coverage reports
39 htmlcov/
40 .tox/
41 .coverage
42 .coverage.*
43 .cache
44 nosetests.xml
45 coverage.xml
46 *,cover
47 .hypothesis/
48
49 # Translations
50 *.mo
51 *.pot
52
53 # Django stuff:
54 *.log
55 local_settings.py
56
57 # Flask stuff:
58 instance/
59 .webassets-cache
60
61 # Scrapy stuff:
62 .scrapy
63
64 # Sphinx documentation
65 docs/_build/
66
67 # PyBuilder
68 target/
69
70 # Jupyter Notebook
71 .ipynb_checkpoints
72
73 # pyenv
74 .python-version
75
76 # celery beat schedule file
77 celerybeat-schedule
78
79 # SageMath parsed files
80 *.sage.py
81
82 # dotenv
83 .env
84
85 # virtualenv
86 .venv
87 venv/
88 ENV/
89
90 # Spyder project settings
91 .spyderproject
92 .spyproject
93
94 # Rope project settings
95 .ropeproject
96
97 # mkdocs documentation
98 /site
7 99
8 100 # eclipse
9 101 .project
10 102 .pydevproject
103 .svn/
104 *.png
105 *.pyc
106 schainpy/scripts
11 107
12 # vscode
13 .vscode No newline at end of file
108 schaingui/node_modules No newline at end of file
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
Show file before | Show file after | Hide comments
@@ -1,179 +1,179
1 1 import threading
2 2 from Queue import Queue
3 3
4 4 from schainpy.controller import Project
5 5 from schainpy.model.graphics.jroplotter import PlotManager
6 6
7 7 class ControllerThread(threading.Thread, Project):
8 8
9 9 def __init__(self, plotter_queue=None):
10 10
11 11 threading.Thread.__init__(self)
12 12 Project.__init__(self, plotter_queue)
13 13
14 14 self.setDaemon(True)
15 15
16 16 self.lock = threading.Lock()
17 17 self.control = {'stop':False, 'pause':False}
18 18
19 19 def __del__(self):
20 20
21 21 self.control['stop'] = True
22 22
23 23 def stop(self):
24 24
25 25 self.lock.acquire()
26 26
27 27 self.control['stop'] = True
28 28
29 29 self.lock.release()
30 30
31 31 def pause(self):
32 32
33 33 self.lock.acquire()
34 34
35 35 self.control['pause'] = not(self.control['pause'])
36 36 paused = self.control['pause']
37 37
38 38 self.lock.release()
39 39
40 40 return paused
41 41
42 42 def isPaused(self):
43 43
44 44 self.lock.acquire()
45 45 paused = self.control['pause']
46 46 self.lock.release()
47 47
48 48 return paused
49 49
50 50 def isStopped(self):
51 51
52 52 self.lock.acquire()
53 53 stopped = self.control['stop']
54 54 self.lock.release()
55 55
56 56 return stopped
57 57
58 58 def run(self):
59 59 self.control['stop'] = False
60 60 self.control['pause'] = False
61 61
62 62 self.writeXml()
63 63
64 64 self.createObjects()
65 65 self.connectObjects()
66 66 Project.run(self)
67 67
68 68 def isRunning(self):
69 69
70 70 return self.is_alive()
71 71
72 72 def isFinished(self):
73 73
74 74 return not self.is_alive()
75 75
76 76 def setPlotters(self):
77 77
78 78 plotterList = PlotManager.plotterList
79 79
80 80 for thisPUConfObj in self.procUnitConfObjDict.values():
81 81
82 82 inputId = thisPUConfObj.getInputId()
83 83
84 84 if int(inputId) == 0:
85 85 continue
86 86
87 87 for thisOpObj in thisPUConfObj.getOperationObjList():
88 88
89 89 if thisOpObj.type == "self":
90 90 continue
91 91
92 92 if thisOpObj.name in plotterList:
93 thisOpObj.type = "plotter"
93 thisOpObj.type = "other"
94 94
95 95 def setPlotterQueue(self, plotter_queue):
96 96
97 97 self.plotterQueue = plotter_queue
98 98
99 99 def getPlotterQueue(self):
100 100
101 101 return self.plotterQueue
102 102
103 103 def useExternalPlotter(self):
104 104
105 105 self.plotterQueue = Queue(10)
106 106 self.setPlotters()
107 107
108 108 plotManagerObj = PlotManager(self.plotterQueue)
109 109 plotManagerObj.setController(self)
110 110
111 111 return plotManagerObj
112 112
113 113 # from PyQt4 import QtCore
114 114 # from PyQt4.QtCore import SIGNAL
115 115 #
116 116 # class ControllerQThread(QtCore.QThread, Project):
117 117 #
118 118 # def __init__(self, filename):
119 119 #
120 120 # QtCore.QThread.__init__(self)
121 121 # Project.__init__(self)
122 122 #
123 123 # self.filename = filename
124 124 #
125 125 # self.lock = threading.Lock()
126 126 # self.control = {'stop':False, 'pause':False}
127 127 #
128 128 # def __del__(self):
129 129 #
130 130 # self.control['stop'] = True
131 131 # self.wait()
132 132 #
133 133 # def stop(self):
134 134 #
135 135 # self.lock.acquire()
136 136 #
137 137 # self.control['stop'] = True
138 138 #
139 139 # self.lock.release()
140 140 #
141 141 # def pause(self):
142 142 #
143 143 # self.lock.acquire()
144 144 #
145 145 # self.control['pause'] = not(self.control['pause'])
146 146 # paused = self.control['pause']
147 147 #
148 148 # self.lock.release()
149 149 #
150 150 # return paused
151 151 #
152 152 # def isPaused(self):
153 153 #
154 154 # self.lock.acquire()
155 155 # paused = self.control['pause']
156 156 # self.lock.release()
157 157 #
158 158 # return paused
159 159 #
160 160 # def isStopped(self):
161 161 #
162 162 # self.lock.acquire()
163 163 # stopped = self.control['stop']
164 164 # self.lock.release()
165 165 #
166 166 # return stopped
167 167 #
168 168 # def run(self):
169 169 #
170 170 # self.control['stop'] = False
171 171 # self.control['pause'] = False
172 172 #
173 173 # self.readXml(self.filename)
174 174 # self.createObjects()
175 175 # self.connectObjects()
176 176 # self.emit( SIGNAL( "jobStarted( PyQt_PyObject )" ), 1)
177 177 # Project.run(self)
178 178 # self.emit( SIGNAL( "jobFinished( PyQt_PyObject )" ), 1)
179 179 # No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1008 +1,1012
1 1 '''
2 2 @author: Daniel Suarez
3 3 '''
4 4 import os
5 5 import glob
6 6 import ftplib
7 7
8 8 try:
9 9 import paramiko
10 10 import scp
11 11 except:
12 12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
13 13
14 14 import time
15 15
16 16 import threading
17 17 Thread = threading.Thread
18 18
19 19 # try:
20 20 # from gevent import sleep
21 21 # except:
22 22 from time import sleep
23 23
24 24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 25
26 26 class Remote(Thread):
27 27 """
28 28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 29 used to upload or download files remotely.
30 30
31 31 Non-standard Python modules used:
32 32 None
33 33
34 34 Written by:
35 35
36 36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
37 37
38 38 """
39 39
40 40 server = None
41 41 username = None
42 42 password = None
43 43 remotefolder = None
44 44
45 45 period = 60
46 46 fileList = []
47 47 bussy = False
48 48
49 49 def __init__(self, server, username, password, remotefolder, period=60):
50 50
51 51 Thread.__init__(self)
52 52
53 53 self.setDaemon(True)
54 54
55 55 self.status = 0
56 56
57 57 self.__server = server
58 58 self.__username = username
59 59 self.__password = password
60 60 self.__remotefolder = remotefolder
61 61
62 62 self.period = period
63 63
64 64 self.fileList = []
65 65 self.bussy = False
66 66
67 67 self.stopFlag = False
68 68
69 69 print "[Remote Server] Opening server: %s" %self.__server
70 70 if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
71 71 print "[Remote Server] %s server was opened successfully" %self.__server
72 72
73 73 self.close()
74 74
75 75 self.mutex = threading.Lock()
76 76
77 77 def stop(self):
78 78
79 79 self.stopFlag = True
80 80 self.join(10)
81 81
82 82 def open(self):
83 83 """
84 84 Connect to server and create a connection class (FTP or SSH) to remote server.
85 85 """
86 86 raise NotImplementedError, "Implement this method in child class"
87 87
88 88 def close(self):
89 89 """
90 90 Close connection to server
91 91 """
92 92 raise NotImplementedError, "Implement this method in child class"
93 93
94 94 def mkdir(self, remotefolder):
95 95 """
96 96 Create a folder remotely
97 97 """
98 98 raise NotImplementedError, "Implement this method in child class"
99 99
100 100 def cd(self, remotefolder):
101 101 """
102 102 Change working directory in remote server
103 103 """
104 104 raise NotImplementedError, "Implement this method in child class"
105 105
106 106 def download(self, filename, localfolder=None):
107 107 """
108 108 Download a file from server to local host
109 109 """
110 110 raise NotImplementedError, "Implement this method in child class"
111 111
112 112 def sendFile(self, fullfilename):
113 113 """
114 114 sendFile method is used to upload a local file to the current directory in remote server
115 115
116 116 Inputs:
117 117 fullfilename - full path name of local file to store in remote directory
118 118
119 119 Returns:
120 120 0 in error case else 1
121 121 """
122 122 raise NotImplementedError, "Implement this method in child class"
123 123
124 124 def upload(self, fullfilename, remotefolder=None):
125 125 """
126 126 upload method is used to upload a local file to remote directory. This method changes
127 127 working directory before sending a file.
128 128
129 129 Inputs:
130 130 fullfilename - full path name of local file to store in remote directory
131 131
132 132 remotefolder - remote directory
133 133
134 134 Returns:
135 135 0 in error case else 1
136 136 """
137 137 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
138 138
139 139 if not self.status:
140 140 return 0
141 141
142 142 if remotefolder == None:
143 143 remotefolder = self.remotefolder
144 144
145 145 if not self.cd(remotefolder):
146 146 return 0
147 147
148 148 if not self.sendFile(fullfilename):
149 149 print "[Remote Server] Error uploading file %s" %fullfilename
150 150 return 0
151 151
152 152 print "[Remote Server] upload finished successfully"
153 153
154 154 return 1
155 155
156 156 def delete(self, filename):
157 157 """
158 158 Remove a file from remote server
159 159 """
160 160 pass
161 161
162 162 def updateFileList(self, fileList):
163 163 """
164 164 Remove a file from remote server
165 165 """
166 166
167 167 if fileList == self.fileList:
168 168 return 0
169 169
170 170 self.mutex.acquire()
171 171 # init = time.time()
172 172 #
173 173 # while(self.bussy):
174 174 # sleep(0.1)
175 175 # if time.time() - init > 2*self.period:
176 176 # return 0
177 177
178 178 self.fileList = fileList
179 179 self.mutex.release()
180 180 return 1
181 181
182 182 def run(self):
183 183
184 184 if not self.status:
185 185 print "Finishing FTP service"
186 186 return
187 187
188 188 if not self.cd(self.remotefolder):
189 189 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
190 190
191 191 while True:
192 192
193 193 for i in range(self.period):
194 194 if self.stopFlag:
195 195 break
196 196 sleep(1)
197 197
198 198 if self.stopFlag:
199 199 break
200 200
201 201 # self.bussy = True
202 202 self.mutex.acquire()
203 203
204 204 print "[Remote Server] Opening %s" %self.__server
205 205 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
206 206 self.mutex.release()
207 207 continue
208 208
209 209 for thisFile in self.fileList:
210 210 self.upload(thisFile, self.remotefolder)
211 211
212 212 print "[Remote Server] Closing %s" %self.__server
213 213 self.close()
214 214
215 215 self.mutex.release()
216 216 # self.bussy = False
217 217
218 218 print "[Remote Server] Thread stopped successfully"
219 219
220 220 class FTPClient(Remote):
221 221
222 222 __ftpClientObj = None
223 223
224 224 def __init__(self, server, username, password, remotefolder, period=60):
225 225 """
226 226 """
227 227 Remote.__init__(self, server, username, password, remotefolder, period)
228 228
229 229 def open(self, server, username, password, remotefolder):
230 230
231 231 """
232 232 This method is used to set FTP parameters and establish a connection to remote server
233 233
234 234 Inputs:
235 235 server - remote server IP Address
236 236
237 237 username - remote server Username
238 238
239 239 password - remote server password
240 240
241 241 remotefolder - remote server current working directory
242 242
243 243 Return:
244 244 Boolean - Returns 1 if a connection has been established, 0 otherwise
245 245
246 246 Affects:
247 247 self.status - in case of error or fail connection this parameter is set to 0 else 1
248 248
249 249 """
250 250
251 251 if server == None:
252 252 raise ValueError, "FTP server should be defined"
253 253
254 254 if username == None:
255 255 raise ValueError, "FTP username should be defined"
256 256
257 257 if password == None:
258 258 raise ValueError, "FTP password should be defined"
259 259
260 260 if remotefolder == None:
261 261 raise ValueError, "FTP remote folder should be defined"
262 262
263 263 try:
264 264 ftpClientObj = ftplib.FTP(server)
265 265 except ftplib.all_errors, e:
266 266 print "[FTP Server]: FTP server connection fail: %s" %server
267 267 print "[FTP Server]:", e
268 268 self.status = 0
269 269 return 0
270 270
271 271 try:
272 272 ftpClientObj.login(username, password)
273 273 except ftplib.all_errors:
274 274 print "[FTP Server]: FTP username or password are incorrect"
275 275 self.status = 0
276 276 return 0
277 277
278 278 if remotefolder == None:
279 279 remotefolder = ftpClientObj.pwd()
280 280 else:
281 281 try:
282 282 ftpClientObj.cwd(remotefolder)
283 283 except ftplib.all_errors:
284 284 print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
285 285 remotefolder = ftpClientObj.pwd()
286 286
287 287 self.server = server
288 288 self.username = username
289 289 self.password = password
290 290 self.remotefolder = remotefolder
291 291 self.__ftpClientObj = ftpClientObj
292 292 self.status = 1
293 293
294 294 return 1
295 295
296 296 def close(self):
297 297 """
298 298 Close connection to remote server
299 299 """
300 300 if not self.status:
301 301 return 0
302 302
303 303 self.__ftpClientObj.close()
304 304
305 305 def mkdir(self, remotefolder):
306 306 """
307 307 mkdir is used to make a new directory in remote server
308 308
309 309 Input:
310 310 remotefolder - directory name
311 311
312 312 Return:
313 313 0 in error case else 1
314 314 """
315 315 if not self.status:
316 316 return 0
317 317
318 318 try:
319 319 self.__ftpClientObj.mkd(dirname)
320 320 except ftplib.all_errors:
321 321 print "[FTP Server]: Error creating remote folder: %s" %remotefolder
322 322 return 0
323 323
324 324 return 1
325 325
326 326 def cd(self, remotefolder):
327 327 """
328 328 cd is used to change remote working directory on server
329 329
330 330 Input:
331 331 remotefolder - current working directory
332 332
333 333 Affects:
334 334 self.remotefolder
335 335
336 336 Return:
337 337 0 in case of error else 1
338 338 """
339 339 if not self.status:
340 340 return 0
341 341
342 342 if remotefolder == self.remotefolder:
343 343 return 1
344 344
345 345 try:
346 346 self.__ftpClientObj.cwd(remotefolder)
347 347 except ftplib.all_errors:
348 348 print '[FTP Server]: Error changing to %s' %remotefolder
349 349 print '[FTP Server]: Trying to create remote folder'
350 350
351 351 if not self.mkdir(remotefolder):
352 352 print '[FTP Server]: Remote folder could not be created'
353 353 return 0
354 354
355 355 try:
356 356 self.__ftpClientObj.cwd(remotefolder)
357 357 except ftplib.all_errors:
358 358 return 0
359 359
360 360 self.remotefolder = remotefolder
361 361
362 362 return 1
363 363
364 364 def sendFile(self, fullfilename):
365 365
366 366 if not self.status:
367 367 return 0
368 368
369 369 fp = open(fullfilename, 'rb')
370 370
371 371 filename = os.path.basename(fullfilename)
372 372
373 373 command = "STOR %s" %filename
374 374
375 375 try:
376 376 self.__ftpClientObj.storbinary(command, fp)
377 377 except ftplib.all_errors, e:
378 378 print "[FTP Server]:", e
379 379 return 0
380 380
381 381 try:
382 382 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
383 383 except ftplib.all_errors, e:
384 384 print "[FTP Server]:", e
385 385
386 386 fp.close()
387 387
388 388 return 1
389 389
390 390 class SSHClient(Remote):
391 391
392 392 __sshClientObj = None
393 393 __scpClientObj = None
394 394
395 395 def __init__(self, server, username, password, remotefolder, period=60):
396 396 """
397 397 """
398 398 Remote.__init__(self, server, username, password, remotefolder, period)
399 399
400 400 def open(self, server, username, password, remotefolder, port=22):
401 401
402 402 """
403 403 This method is used to set SSH parameters and establish a connection to a remote server
404 404
405 405 Inputs:
406 406 server - remote server IP Address
407 407
408 408 username - remote server Username
409 409
410 410 password - remote server password
411 411
412 412 remotefolder - remote server current working directory
413 413
414 414 Return: void
415 415
416 416 Affects:
417 417 self.status - in case of error or fail connection this parameter is set to 0 else 1
418 418
419 419 """
420 420 import socket
421 421
422 422 if server == None:
423 423 raise ValueError, "SSH server should be defined"
424 424
425 425 if username == None:
426 426 raise ValueError, "SSH username should be defined"
427 427
428 428 if password == None:
429 429 raise ValueError, "SSH password should be defined"
430 430
431 431 if remotefolder == None:
432 432 raise ValueError, "SSH remote folder should be defined"
433 433
434 434 sshClientObj = paramiko.SSHClient()
435 435
436 436 sshClientObj.load_system_host_keys()
437 437 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
438 438
439 439 self.status = 0
440 440 try:
441 441 sshClientObj.connect(server, username=username, password=password, port=port)
442 442 except paramiko.AuthenticationException, e:
443 443 # print "SSH username or password are incorrect: %s"
444 444 print "[SSH Server]:", e
445 445 return 0
446 446 except SSHException, e:
447 447 print "[SSH Server]:", e
448 448 return 0
449 449 except socket.error:
450 450 self.status = 0
451 451 print "[SSH Server]:", e
452 452 return 0
453 453
454 454 self.status = 1
455 455 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
456 456
457 457 if remotefolder == None:
458 458 remotefolder = self.pwd()
459 459
460 460 self.server = server
461 461 self.username = username
462 462 self.password = password
463 463 self.__sshClientObj = sshClientObj
464 464 self.__scpClientObj = scpClientObj
465 465 self.status = 1
466 466
467 467 if not self.cd(remotefolder):
468 468 raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
469 469 return 0
470 470
471 471 self.remotefolder = remotefolder
472 472
473 473 return 1
474 474
475 475 def close(self):
476 476 """
477 477 Close connection to remote server
478 478 """
479 479 if not self.status:
480 480 return 0
481 481
482 482 self.__scpClientObj.close()
483 483 self.__sshClientObj.close()
484 484
485 485 def __execute(self, command):
486 486 """
487 487 __execute a command on remote server
488 488
489 489 Input:
490 490 command - Exmaple 'ls -l'
491 491
492 492 Return:
493 493 0 in error case else 1
494 494 """
495 495 if not self.status:
496 496 return 0
497 497
498 498 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
499 499
500 500 result = stderr.readlines()
501 501 if len(result) > 1:
502 502 return 0
503 503
504 504 result = stdout.readlines()
505 505 if len(result) > 1:
506 506 return result[0][:-1]
507 507
508 508 return 1
509 509
510 510 def mkdir(self, remotefolder):
511 511 """
512 512 mkdir is used to make a new directory in remote server
513 513
514 514 Input:
515 515 remotefolder - directory name
516 516
517 517 Return:
518 518 0 in error case else 1
519 519 """
520 520
521 521 command = 'mkdir %s' %remotefolder
522 522
523 523 return self.__execute(command)
524 524
525 525 def pwd(self):
526 526
527 527 command = 'pwd'
528 528
529 529 return self.__execute(command)
530 530
531 531 def cd(self, remotefolder):
532 532 """
533 533 cd is used to change remote working directory on server
534 534
535 535 Input:
536 536 remotefolder - current working directory
537 537
538 538 Affects:
539 539 self.remotefolder
540 540
541 541 Return:
542 542 0 in case of error else 1
543 543 """
544 544 if not self.status:
545 545 return 0
546 546
547 547 if remotefolder == self.remotefolder:
548 548 return 1
549 549
550 550 chk_command = "cd %s; pwd" %remotefolder
551 551 mkdir_command = "mkdir %s" %remotefolder
552 552
553 553 if not self.__execute(chk_command):
554 554 if not self.__execute(mkdir_command):
555 555 self.remotefolder = None
556 556 return 0
557 557
558 558 self.remotefolder = remotefolder
559 559
560 560 return 1
561 561
562 562 def sendFile(self, fullfilename):
563 563
564 564 if not self.status:
565 565 return 0
566 566
567 567 try:
568 568 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
569 569 except scp.ScpError, e:
570 570 print "[SSH Server]", str(e)
571 571 return 0
572 572
573 573 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
574 574 command = 'chmod 775 %s' %remotefile
575 575
576 576 return self.__execute(command)
577 577
578 578 class SendToServer(ProcessingUnit):
579 579
580 580 def __init__(self):
581 581
582 582 ProcessingUnit.__init__(self)
583 583
584 584 self.isConfig = False
585 585 self.clientObj = None
586 586
587
588
587 589 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
588 590
589 591 self.clientObj = None
590 592 self.localfolder = localfolder
591 593 self.ext = ext
592 594 self.period = period
593 595
594 596 if str.lower(protocol) == 'ftp':
595 597 self.clientObj = FTPClient(server, username, password, remotefolder, period)
596 598
597 599 if str.lower(protocol) == 'ssh':
598 600 self.clientObj = SSHClient(server, username, password, remotefolder, period)
599 601
600 602 if not self.clientObj:
601 603 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
602 604
603 605 self.clientObj.start()
604 606
605 607 def findFiles(self):
606 608
607 609 if not type(self.localfolder) == list:
608 610 folderList = [self.localfolder]
609 611 else:
610 612 folderList = self.localfolder
611 613
612 614 #Remove duplicate items
613 615 folderList = list(set(folderList))
614 616
615 617 fullfilenameList = []
616 618
617 619 for thisFolder in folderList:
618 620
619 621 print "[Remote Server]: Searching files on %s" %thisFolder
620 622
621 623 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
622 624
623 625 if len(filenameList) < 1:
624 626 continue
625 627
626 628 for thisFile in filenameList:
627 629 fullfilename = os.path.join(thisFolder, thisFile)
628 630
629 631 if fullfilename in fullfilenameList:
630 632 continue
631 633
632 634 #Only files modified in the last 30 minutes are considered
633 635 if os.path.getmtime(fullfilename) < time.time() - 30*60:
634 636 continue
635 637
636 638 fullfilenameList.append(fullfilename)
637 639
638 640 return fullfilenameList
639 641
640 642 def run(self, **kwargs):
641
642 643 if not self.isConfig:
643 644 self.init = time.time()
644 645 self.setup(**kwargs)
645 646 self.isConfig = True
646 647
648 if not self.clientObj.is_alive():
649 print "[Remote Server]: Restarting connection "
650 self.setup(**kwargs)
651
647 652 if time.time() - self.init >= self.period:
648 653 fullfilenameList = self.findFiles()
649 654
650 655 if self.clientObj.updateFileList(fullfilenameList):
651 656 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
652
653 657 self.init = time.time()
654 658
655 659 def close(self):
656 660 print "[Remote Server] Stopping thread"
657 661 self.clientObj.stop()
658 662
659 663
660 664 class FTP(object):
661 665 """
662 666 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
663 667
664 668 Non-standard Python modules used: None
665 669
666 670 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
667 671 """
668 672
669 673 def __init__(self,server = None, username=None, password=None, remotefolder=None):
670 674 """
671 675 This method is used to setting parameters for FTP and establishing connection to remote server
672 676
673 677 Inputs:
674 678 server - remote server IP Address
675 679
676 680 username - remote server Username
677 681
678 682 password - remote server password
679 683
680 684 remotefolder - remote server current working directory
681 685
682 686 Return: void
683 687
684 688 Affects:
685 689 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
686 690
687 691 self.folderList - sub-folder list of remote folder
688 692
689 693 self.fileList - file list of remote folder
690 694
691 695
692 696 """
693 697
694 698 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
695 699 server, username, password, remotefolder = self.parmsByDefault()
696 700
697 701 self.server = server
698 702 self.username = username
699 703 self.password = password
700 704 self.remotefolder = remotefolder
701 705 self.file = None
702 706 self.ftp = None
703 707 self.status = 0
704 708
705 709 try:
706 710 self.ftp = ftplib.FTP(self.server)
707 711 self.ftp.login(self.username,self.password)
708 712 self.ftp.cwd(self.remotefolder)
709 713 # print 'Connect to FTP Server: Successfully'
710 714
711 715 except ftplib.all_errors:
712 716 print 'Error FTP Service'
713 717 self.status = 1
714 718 return
715 719
716 720
717 721
718 722 self.dirList = []
719 723
720 724 try:
721 725 self.dirList = self.ftp.nlst()
722 726
723 727 except ftplib.error_perm, resp:
724 728 if str(resp) == "550 No files found":
725 729 print "no files in this directory"
726 730 self.status = 1
727 731 return
728 732
729 733 except ftplib.all_errors:
730 734 print 'Error Displaying Dir-Files'
731 735 self.status = 1
732 736 return
733 737
734 738 self.fileList = []
735 739 self.folderList = []
736 740 #only for test
737 741 for f in self.dirList:
738 742 name, ext = os.path.splitext(f)
739 743 if ext != '':
740 744 self.fileList.append(f)
741 745 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
742 746
743 747 def parmsByDefault(self):
744 748 server = 'jro-app.igp.gob.pe'
745 749 username = 'wmaster'
746 750 password = 'mst2010vhf'
747 751 remotefolder = '/home/wmaster/graficos'
748 752
749 753 return server, username, password, remotefolder
750 754
751 755
752 756 def mkd(self,dirname):
753 757 """
754 758 mkd is used to make directory in remote server
755 759
756 760 Input:
757 761 dirname - directory name
758 762
759 763 Return:
760 764 1 in error case else 0
761 765 """
762 766 try:
763 767 self.ftp.mkd(dirname)
764 768 except:
765 769 print 'Error creating remote folder:%s'%dirname
766 770 return 1
767 771
768 772 return 0
769 773
770 774
771 775 def delete(self,filename):
772 776 """
773 777 delete is used to delete file in current working directory of remote server
774 778
775 779 Input:
776 780 filename - filename to delete in remote folder
777 781
778 782 Return:
779 783 1 in error case else 0
780 784 """
781 785
782 786 try:
783 787 self.ftp.delete(filename)
784 788 except:
785 789 print 'Error deleting remote file:%s'%filename
786 790 return 1
787 791
788 792 return 0
789 793
790 794 def download(self,filename,localfolder):
791 795 """
792 796 download is used to downloading file from remote folder into local folder
793 797
794 798 Inputs:
795 799 filename - filename to donwload
796 800
797 801 localfolder - directory local to store filename
798 802
799 803 Returns:
800 804 self.status - 1 in error case else 0
801 805 """
802 806
803 807 self.status = 0
804 808
805 809
806 810 if not(filename in self.fileList):
807 811 print 'filename:%s not exists'%filename
808 812 self.status = 1
809 813 return self.status
810 814
811 815 newfilename = os.path.join(localfolder,filename)
812 816
813 817 self.file = open(newfilename, 'wb')
814 818
815 819 try:
816 820 print 'Download: ' + filename
817 821 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
818 822 print 'Download Complete'
819 823 except ftplib.all_errors:
820 824 print 'Error Downloading ' + filename
821 825 self.status = 1
822 826 return self.status
823 827
824 828 self.file.close()
825 829
826 830 return self.status
827 831
828 832
829 833 def __handleDownload(self,block):
830 834 """
831 835 __handleDownload is used to handle writing file
832 836 """
833 837 self.file.write(block)
834 838
835 839
836 840 def upload(self,filename,remotefolder=None):
837 841 """
838 842 upload is used to uploading local file to remote directory
839 843
840 844 Inputs:
841 845 filename - full path name of local file to store in remote directory
842 846
843 847 remotefolder - remote directory
844 848
845 849 Returns:
846 850 self.status - 1 in error case else 0
847 851 """
848 852
849 853 if remotefolder == None:
850 854 remotefolder = self.remotefolder
851 855
852 856 self.status = 0
853 857
854 858 try:
855 859 self.ftp.cwd(remotefolder)
856 860
857 861 self.file = open(filename, 'rb')
858 862
859 863 (head, tail) = os.path.split(filename)
860 864
861 865 command = "STOR " + tail
862 866
863 867 print 'Uploading: ' + tail
864 868 self.ftp.storbinary(command, self.file)
865 869 print 'Upload Completed'
866 870
867 871 except ftplib.all_errors:
868 872 print 'Error Uploading ' + tail
869 873 self.status = 1
870 874 return self.status
871 875
872 876 self.file.close()
873 877
874 878 #back to initial directory in __init__()
875 879 self.ftp.cwd(self.remotefolder)
876 880
877 881 return self.status
878 882
879 883
880 884 def dir(self,remotefolder):
881 885 """
882 886 dir is used to change working directory of remote server and get folder and file list
883 887
884 888 Input:
885 889 remotefolder - current working directory
886 890
887 891 Affects:
888 892 self.fileList - file list of working directory
889 893
890 894 Return:
891 895 infoList - list with filenames and size of file in bytes
892 896
893 897 self.folderList - folder list
894 898 """
895 899
896 900 self.remotefolder = remotefolder
897 901 print 'Change to ' + self.remotefolder
898 902 try:
899 903 self.ftp.cwd(remotefolder)
900 904 except ftplib.all_errors:
901 905 print 'Error Change to ' + self.remotefolder
902 906 infoList = None
903 907 self.folderList = None
904 908 return infoList,self.folderList
905 909
906 910 self.dirList = []
907 911
908 912 try:
909 913 self.dirList = self.ftp.nlst()
910 914
911 915 except ftplib.error_perm, resp:
912 916 if str(resp) == "550 No files found":
913 917 print "no files in this directory"
914 918 infoList = None
915 919 self.folderList = None
916 920 return infoList,self.folderList
917 921 except ftplib.all_errors:
918 922 print 'Error Displaying Dir-Files'
919 923 infoList = None
920 924 self.folderList = None
921 925 return infoList,self.folderList
922 926
923 927 infoList = []
924 928 self.fileList = []
925 929 self.folderList = []
926 930 for f in self.dirList:
927 931 name,ext = os.path.splitext(f)
928 932 if ext != '':
929 933 self.fileList.append(f)
930 934 value = (f,self.ftp.size(f))
931 935 infoList.append(value)
932 936
933 937 if ext == '':
934 938 self.folderList.append(f)
935 939
936 940 return infoList,self.folderList
937 941
938 942
939 943 def close(self):
940 944 """
941 945 close is used to close and end FTP connection
942 946
943 947 Inputs: None
944 948
945 949 Return: void
946 950
947 951 """
948 952 self.ftp.close()
949 953
950 954 class SendByFTP(Operation):
951 955
952 956 def __init__(self):
953 957
954 958 self.status = 1
955 959 self.counter = 0
956 960
957 961 def error_print(self, ValueError):
958 962
959 963 print ValueError, 'Error FTP'
960 964 print "don't worry the program is running..."
961 965
962 966 def worker_ftp(self, server, username, password, remotefolder, filenameList):
963 967
964 968 self.ftpClientObj = FTP(server, username, password, remotefolder)
965 969 for filename in filenameList:
966 970 self.ftpClientObj.upload(filename)
967 971 self.ftpClientObj.close()
968 972
969 973 def ftp_thread(self, server, username, password, remotefolder):
970 974 if not(self.status):
971 975 return
972 976
973 977 import multiprocessing
974 978
975 979 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
976 980 p.start()
977 981
978 982 p.join(3)
979 983
980 984 if p.is_alive():
981 985 p.terminate()
982 986 p.join()
983 987 print 'killing ftp process...'
984 988 self.status = 0
985 989 return
986 990
987 991 self.status = 1
988 992 return
989 993
990 994 def filterByExt(self, ext, localfolder):
991 995 fnameList = glob.glob1(localfolder,ext)
992 996 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
993 997
994 998 if len(self.filenameList) == 0:
995 999 self.status = 0
996 1000
997 1001 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
998 1002
999 1003 self.counter += 1
1000 1004 if self.counter >= period:
1001 1005 self.filterByExt(ext, localfolder)
1002 1006
1003 1007 self.ftp_thread(server, username, password, remotefolder)
1004 1008
1005 1009 self.counter = 0
1006 1010
1007 1011 self.status = 1
1008 1012
General Comments 0
You need to be logged in to leave comments. Login now