##// END OF EJS Templates
schain
RSS
jroutils_ftp.py: SSHClient changes file permissions on remote server after upload a file
Miguel Valdez -
r576:58b2c6816db8
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,912 +1,940
1 '''
1 '''
2 @author: Daniel Suarez
2 @author: Daniel Suarez
3 '''
3 '''
4 import os
4 import os
5 import glob
5 import glob
6 import ftplib
6 import ftplib
7
7
8 try:
8 try:
9 import paramiko
9 import paramiko
10 import scp
10 import scp
11 except:
11 except:
12 print "You should install paramiko if you will use SSH protocol to upload files to a server"
12 print "You should install paramiko if you will use SSH protocol to upload files to a server"
13
13
14 import multiprocessing
14 import multiprocessing
15
15
16 import time
16 import time
17 import threading
17 import threading
18
18
19
19
20 try:
20 try:
21 from gevent import sleep
21 from gevent import sleep
22 except:
22 except:
23 from time import sleep
23 from time import sleep
24
24
25 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
26
26
27 class Remote(threading.Thread):
27 class Remote(threading.Thread):
28 """
28 """
29 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
30 used to upload or download files remotely.
30 used to upload or download files remotely.
31
31
32 Non-standard Python modules used:
32 Non-standard Python modules used:
33 None
33 None
34
34
35 Written by:
35 Written by:
36
36
37 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
37 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
38
38
39 """
39 """
40
40
41 server = None
41 server = None
42 username = None
42 username = None
43 password = None
43 password = None
44 remotefolder = None
44 remotefolder = None
45
45
46 period = 60
46 period = 60
47 fileList = []
47 fileList = []
48 bussy = False
48 bussy = False
49
49
50 def __init__(self, server, username, password, remotefolder, period=60):
50 def __init__(self, server, username, password, remotefolder, period=60):
51
51
52 threading.Thread.__init__(self)
52 threading.Thread.__init__(self)
53 self._stop = threading.Event()
53 self._stop = threading.Event()
54
54
55 self.status = 0
55 self.status = 0
56
56
57 self.period = period
57 self.period = period
58 self.fileList = []
58 self.fileList = []
59 self.bussy = False
59 self.bussy = False
60
60
61 self.stopFlag = False
61 self.stopFlag = False
62
62
63 print "[Remote Server] Opening server: %s" %server
63 print "[Remote Server] Opening server: %s" %server
64 if self.open(server, username, password, remotefolder):
64 if self.open(server, username, password, remotefolder):
65 print "[Remote Server] %s server was opened successfully" %server
65 print "[Remote Server] %s server was opened successfully" %server
66
66
67 def stop(self):
67 def stop(self):
68
68
69 self.stopFlag = True
69 self.stopFlag = True
70
70
71 def open(self):
71 def open(self):
72 """
72 """
73 Connect to server and create a connection class (FTP or SSH) to remote server.
73 Connect to server and create a connection class (FTP or SSH) to remote server.
74 """
74 """
75 raise NotImplementedError, "Implement this method in child class"
75 raise NotImplementedError, "Implement this method in child class"
76
76
77 def close(self):
77 def close(self):
78 """
78 """
79 Close connection to server
79 Close connection to server
80 """
80 """
81 raise NotImplementedError, "Implement this method in child class"
81 raise NotImplementedError, "Implement this method in child class"
82
82
83 def mkdir(self, remotefolder):
83 def mkdir(self, remotefolder):
84 """
84 """
85 Create a folder remotely
85 Create a folder remotely
86 """
86 """
87 raise NotImplementedError, "Implement this method in child class"
87 raise NotImplementedError, "Implement this method in child class"
88
88
89 def cd(self, remotefolder):
89 def cd(self, remotefolder):
90 """
90 """
91 Change working directory in remote server
91 Change working directory in remote server
92 """
92 """
93 raise NotImplementedError, "Implement this method in child class"
93 raise NotImplementedError, "Implement this method in child class"
94
94
95 def download(self, filename, localfolder=None):
95 def download(self, filename, localfolder=None):
96 """
96 """
97 Download a file from server to local host
97 Download a file from server to local host
98 """
98 """
99 raise NotImplementedError, "Implement this method in child class"
99 raise NotImplementedError, "Implement this method in child class"
100
100
101 def sendFile(self, fullfilename):
101 def sendFile(self, fullfilename):
102 """
102 """
103 sendFile method is used to upload a local file to the current directory in remote server
103 sendFile method is used to upload a local file to the current directory in remote server
104
104
105 Inputs:
105 Inputs:
106 fullfilename - full path name of local file to store in remote directory
106 fullfilename - full path name of local file to store in remote directory
107
107
108 Returns:
108 Returns:
109 0 in error case else 1
109 0 in error case else 1
110 """
110 """
111 raise NotImplementedError, "Implement this method in child class"
111 raise NotImplementedError, "Implement this method in child class"
112
112
113 def upload(self, fullfilename, remotefolder=None):
113 def upload(self, fullfilename, remotefolder=None):
114 """
114 """
115 upload method is used to upload a local file to remote directory. This method changes
115 upload method is used to upload a local file to remote directory. This method changes
116 working directory before sending a file.
116 working directory before sending a file.
117
117
118 Inputs:
118 Inputs:
119 fullfilename - full path name of local file to store in remote directory
119 fullfilename - full path name of local file to store in remote directory
120
120
121 remotefolder - remote directory
121 remotefolder - remote directory
122
122
123 Returns:
123 Returns:
124 0 in error case else 1
124 0 in error case else 1
125 """
125 """
126 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
126 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
127
127
128 if not self.status:
128 if not self.status:
129 return 0
129 return 0
130
130
131 if remotefolder == None:
131 if remotefolder == None:
132 remotefolder = self.remotefolder
132 remotefolder = self.remotefolder
133
133
134 if not self.cd(remotefolder):
134 if not self.cd(remotefolder):
135 return 0
135 return 0
136
136
137 if not self.sendFile(fullfilename):
137 if not self.sendFile(fullfilename):
138 print "[Remote Server] Error uploading file %s" %fullfilename
138 print "[Remote Server] Error uploading file %s" %fullfilename
139 return 0
139 return 0
140
140
141 print "[Remote Server] upload finished successfully"
141 print "[Remote Server] upload finished successfully"
142
142
143 return 1
143 return 1
144
144
145 def delete(self, filename):
145 def delete(self, filename):
146 """
146 """
147 Remove a file from remote server
147 Remove a file from remote server
148 """
148 """
149 pass
149 pass
150
150
151 def updateFileList(self, fileList):
151 def updateFileList(self, fileList):
152 """
152 """
153 Remove a file from remote server
153 Remove a file from remote server
154 """
154 """
155
155
156 if fileList == self.fileList:
156 if fileList == self.fileList:
157 return 1
157 return 1
158
158
159 init = time.time()
159 init = time.time()
160
160
161 while(self.bussy):
161 while(self.bussy):
162 sleep(0.1)
162 sleep(0.1)
163 if time.time() - init > 2*self.period:
163 if time.time() - init > 2*self.period:
164 return 0
164 return 0
165
165
166 self.fileList = fileList
166 self.fileList = fileList
167
167
168 return 1
168 return 1
169
169
170 def run(self):
170 def run(self):
171
171
172 if not self.cd(self.remotefolder):
172 if not self.cd(self.remotefolder):
173 raise ValueError, "It could not change to the new remote directory: %s" %remotefolder
173 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
174
174
175 while True:
175 while True:
176
176
177 sleep(self.period)
177 sleep(self.period)
178
178
179 self.bussy = True
179 self.bussy = True
180
180
181 for thisFile in self.fileList:
181 for thisFile in self.fileList:
182 self.upload(thisFile, self.remotefolder)
182 self.upload(thisFile, self.remotefolder)
183
183
184 self.bussy = False
184 self.bussy = False
185
185
186 if self.stopFlag:
186 if self.stopFlag:
187 break
187 break
188
188
189 print "[Remote Server] Thread stopped successfully"
189 print "[Remote Server] Thread stopped successfully"
190
190
191 class FTPClient(Remote):
191 class FTPClient(Remote):
192
192
193 __ftpClientObj = None
193 __ftpClientObj = None
194
194
195 def __init__(self, server, username, password, remotefolder, period=60):
195 def __init__(self, server, username, password, remotefolder, period=60):
196 """
196 """
197 """
197 """
198 Remote.__init__(self, server, username, password, remotefolder, period)
198 Remote.__init__(self, server, username, password, remotefolder, period)
199
199
200 def open(self, server, username, password, remotefolder):
200 def open(self, server, username, password, remotefolder):
201
201
202 """
202 """
203 This method is used to set FTP parameters and establish a connection to remote server
203 This method is used to set FTP parameters and establish a connection to remote server
204
204
205 Inputs:
205 Inputs:
206 server - remote server IP Address
206 server - remote server IP Address
207
207
208 username - remote server Username
208 username - remote server Username
209
209
210 password - remote server password
210 password - remote server password
211
211
212 remotefolder - remote server current working directory
212 remotefolder - remote server current working directory
213
213
214 Return: void
214 Return: void
215
215
216 Affects:
216 Affects:
217 self.status - in case of error or fail connection this parameter is set to 0 else 1
217 self.status - in case of error or fail connection this parameter is set to 0 else 1
218
218
219 """
219 """
220
220
221 if server == None:
221 if server == None:
222 raise ValueError, "FTP server should be defined"
222 raise ValueError, "FTP server should be defined"
223
223
224 if username == None:
224 if username == None:
225 raise ValueError, "FTP username should be defined"
225 raise ValueError, "FTP username should be defined"
226
226
227 if password == None:
227 if password == None:
228 raise ValueError, "FTP password should be defined"
228 raise ValueError, "FTP password should be defined"
229
229
230 if remotefolder == None:
230 if remotefolder == None:
231 raise ValueError, "FTP remote folder should be defined"
231 raise ValueError, "FTP remote folder should be defined"
232
232
233 try:
233 try:
234 ftpClientObj = ftplib.FTP(server)
234 ftpClientObj = ftplib.FTP(server)
235 except ftplib.all_errors:
235 except ftplib.all_errors:
236 print "FTP server connection fail: %s" %server
236 print "FTP server connection fail: %s" %server
237 self.status = 0
237 self.status = 0
238 return 0
238 return 0
239
239
240 try:
240 try:
241 ftpClientObj.login(username, password)
241 ftpClientObj.login(username, password)
242 except ftplib.all_errors:
242 except ftplib.all_errors:
243 print "FTP username or password are incorrect"
243 print "FTP username or password are incorrect"
244 self.status = 0
244 self.status = 0
245 return 0
245 return 0
246
246
247 if remotefolder == None:
247 if remotefolder == None:
248 remotefolder = ftpClientObj.pwd()
248 remotefolder = ftpClientObj.pwd()
249 else:
249 else:
250 try:
250 try:
251 ftpClientObj.cwd(remotefolder)
251 ftpClientObj.cwd(remotefolder)
252 except ftplib.all_errors:
252 except ftplib.all_errors:
253 print "FTP remote folder is invalid: %s" %remotefolder
253 print "FTP remote folder is invalid: %s" %remotefolder
254 remotefolder = ftpClientObj.pwd()
254 remotefolder = ftpClientObj.pwd()
255
255
256 self.server = server
256 self.server = server
257 self.username = username
257 self.username = username
258 self.password = password
258 self.password = password
259 self.remotefolder = remotefolder
259 self.remotefolder = remotefolder
260 self.__ftpClientObj = ftpClientObj
260 self.__ftpClientObj = ftpClientObj
261 self.status = 1
261 self.status = 1
262
262
263 return 1
263 return 1
264
264
265 def close(self):
265 def close(self):
266 """
266 """
267 Close connection to remote server
267 Close connection to remote server
268 """
268 """
269 if not self.status:
269 if not self.status:
270 return 0
270 return 0
271
271
272 self.__ftpClientObj.close()
272 self.__ftpClientObj.close()
273
273
274 def mkdir(self, remotefolder):
274 def mkdir(self, remotefolder):
275 """
275 """
276 mkdir is used to make a new directory in remote server
276 mkdir is used to make a new directory in remote server
277
277
278 Input:
278 Input:
279 remotefolder - directory name
279 remotefolder - directory name
280
280
281 Return:
281 Return:
282 0 in error case else 1
282 0 in error case else 1
283 """
283 """
284 if not self.status:
284 if not self.status:
285 return 0
285 return 0
286
286
287 try:
287 try:
288 self.__ftpClientObj.mkd(dirname)
288 self.__ftpClientObj.mkd(dirname)
289 except ftplib.all_errors:
289 except ftplib.all_errors:
290 print "Error creating remote folder: %s" %remotefolder
290 print "Error creating remote folder: %s" %remotefolder
291 return 0
291 return 0
292
292
293 return 1
293 return 1
294
294
295 def cd(self, remotefolder):
295 def cd(self, remotefolder):
296 """
296 """
297 cd is used to change remote working directory on server
297 cd is used to change remote working directory on server
298
298
299 Input:
299 Input:
300 remotefolder - current working directory
300 remotefolder - current working directory
301
301
302 Affects:
302 Affects:
303 self.remotefolder
303 self.remotefolder
304
304
305 Return:
305 Return:
306 0 in case of error else 1
306 0 in case of error else 1
307 """
307 """
308 if not self.status:
308 if not self.status:
309 return 0
309 return 0
310
310
311 if remotefolder == self.remotefolder:
311 if remotefolder == self.remotefolder:
312 return 1
312 return 1
313
313
314 try:
314 try:
315 self.__ftpClientObj.cwd(remotefolder)
315 self.__ftpClientObj.cwd(remotefolder)
316 except ftplib.all_errors:
316 except ftplib.all_errors:
317 print 'Error changing to %s' %remotefolder
317 print 'Error changing to %s' %remotefolder
318 print 'Trying to create remote folder'
318 print 'Trying to create remote folder'
319
319
320 if not self.mkdir(remotefolder):
320 if not self.mkdir(remotefolder):
321 print 'Remote folder could not be created'
321 print 'Remote folder could not be created'
322 return 0
322 return 0
323
323
324 try:
324 try:
325 self.__ftpClientObj.cwd(remotefolder)
325 self.__ftpClientObj.cwd(remotefolder)
326 except ftplib.all_errors:
326 except ftplib.all_errors:
327 return 0
327 return 0
328
328
329 self.remotefolder = remotefolder
329 self.remotefolder = remotefolder
330
330
331 return 1
331 return 1
332
332
333 def sendFile(self, fullfilename):
333 def sendFile(self, fullfilename):
334
334
335 if not self.status:
335 if not self.status:
336 return 0
336 return 0
337
337
338 file = open(fullfilename, 'rb')
338 file = open(fullfilename, 'rb')
339
339
340 filename = os.path.split(fullfilename)[-1]
340 filename = os.path.split(fullfilename)[-1]
341
341
342 command = "STOR %s" %filename
342 command = "STOR %s" %filename
343
343
344 try:
344 try:
345 self.__ftpClientObj.storbinary(command, file)
345 self.__ftpClientObj.storbinary(command, file)
346 except ftplib.all_errors:
346 except ftplib.all_errors:
347 return 0
347 return 0
348
348
349 try:
349 try:
350 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
350 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
351 except ftplib.all_errors, e:
351 except ftplib.all_errors, e:
352 print e
352 print e
353
353
354 file.close()
354 file.close()
355
355
356 return 1
356 return 1
357
357
358 class SSHClient(Remote):
358 class SSHClient(Remote):
359
359
360 __sshClientObj = None
360 __sshClientObj = None
361 __scpClientObj = None
361 __scpClientObj = None
362
362
363 def __init__(self, server, username, password, remotefolder, period=60):
363 def __init__(self, server, username, password, remotefolder, period=60):
364 """
364 """
365 """
365 """
366 Remote.__init__(self, server, username, password, remotefolder, period)
366 Remote.__init__(self, server, username, password, remotefolder, period)
367
367
368 def open(self, server, username, password, remotefolder, port=22):
368 def open(self, server, username, password, remotefolder, port=22):
369
369
370 """
370 """
371 This method is used to set SSH parameters and establish a connection to a remote server
371 This method is used to set SSH parameters and establish a connection to a remote server
372
372
373 Inputs:
373 Inputs:
374 server - remote server IP Address
374 server - remote server IP Address
375
375
376 username - remote server Username
376 username - remote server Username
377
377
378 password - remote server password
378 password - remote server password
379
379
380 remotefolder - remote server current working directory
380 remotefolder - remote server current working directory
381
381
382 Return: void
382 Return: void
383
383
384 Affects:
384 Affects:
385 self.status - in case of error or fail connection this parameter is set to 0 else 1
385 self.status - in case of error or fail connection this parameter is set to 0 else 1
386
386
387 """
387 """
388
388
389 if server == None:
389 if server == None:
390 raise ValueError, "SSH server should be defined"
390 raise ValueError, "SSH server should be defined"
391
391
392 if username == None:
392 if username == None:
393 raise ValueError, "SSH username should be defined"
393 raise ValueError, "SSH username should be defined"
394
394
395 if password == None:
395 if password == None:
396 raise ValueError, "SSH password should be defined"
396 raise ValueError, "SSH password should be defined"
397
397
398 if remotefolder == None:
398 if remotefolder == None:
399 raise ValueError, "SSH remote folder should be defined"
399 raise ValueError, "SSH remote folder should be defined"
400
400
401 try:
401 try:
402 sshClientObj = paramiko.SSHClient()
402 sshClientObj = paramiko.SSHClient()
403 except:
403 except:
404 print "SSH server connection fail: %s" %server
404 print "SSH server connection fail: %s" %server
405 self.status = 0
405 self.status = 0
406 return 0
406 return 0
407
407
408 sshClientObj.load_system_host_keys()
408 sshClientObj.load_system_host_keys()
409 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
409 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
410
410
411 try:
411 try:
412 sshClientObj.connect(server, username=username, password=password, port=port)
412 sshClientObj.connect(server, username=username, password=password, port=port)
413 except :
413 except :
414 print "SSH username or password are incorrect: %s"
414 print "SSH username or password are incorrect: %s"
415 self.status = 0
415 self.status = 0
416 return 0
416 return 0
417
417
418 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
418 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
419
419
420 if remotefolder == None:
420 if remotefolder == None:
421 remotefolder = self.pwd()
421 remotefolder = self.pwd()
422
422
423 self.server = server
423 self.server = server
424 self.username = username
424 self.username = username
425 self.password = password
425 self.password = password
426 self.remotefolder = remotefolder
427 self.__sshClientObj = sshClientObj
426 self.__sshClientObj = sshClientObj
428 self.__scpClientObj = scpClientObj
427 self.__scpClientObj = scpClientObj
429 self.status = 1
428 self.status = 1
430
429
430 if not self.cd(remotefolder):
431 raise ValueError, "Could not access to remote folder: %s" %remotefolder
432 return 0
433
434 self.remotefolder = remotefolder
435
431 return 1
436 return 1
432
437
433 def close(self):
438 def close(self):
434 """
439 """
435 Close connection to remote server
440 Close connection to remote server
436 """
441 """
437 if not self.status:
442 if not self.status:
438 return 0
443 return 0
439
444
440 self.__sshObj.close()
445 self.__sshObj.close()
441
446
442 def mkdir(self, remotefolder):
447 def __execute(self, command):
443 """
448 """
444 mkdir is used to make a new directory in remote server
449 __execute a command on remote server
445
450
446 Input:
451 Input:
447 remotefolder - directory name
452 command - Exmaple 'ls -l'
448
453
449 Return:
454 Return:
450 0 in error case else 1
455 0 in error case else 1
451 """
456 """
452 if not self.status:
457 if not self.status:
453 return 0
458 return 0
454
459
455 stdin, stdout, stderr = self.__sshClientObj.exec_command('mkdir %s' %remotefolder)
460 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
456 result = stderr.readlines()[0]
457
461
462 result = stderr.readlines()
458 if len(result) > 1:
463 if len(result) > 1:
459 return 0
464 return 0
460
465
466 result = stdout.readlines()
467 if len(result) > 1:
468 return result[0][:-1]
469
461 return 1
470 return 1
462
471
463 def pwd(self):
472 def mkdir(self, remotefolder):
473 """
474 mkdir is used to make a new directory in remote server
464
475
465 if not self.status:
476 Input:
466 return None
477 remotefolder - directory name
478
479 Return:
480 0 in error case else 1
481 """
467
482
468 stdin, stdout, stderr = self.__sshClientObj.exec_command('pwd')
483 command = 'mkdir %s' %remotefolder
469 result = stdout.readlines()[0]
470
484
471 if len(result) < 1:
485 return self.__execute(command)
472 return None
473
486
474 return result[:-1]
487 def pwd(self):
488
489 command = 'pwd'
490
491 return self.__execute(command)
475
492
476 def cd(self, remotefolder):
493 def cd(self, remotefolder):
477 """
494 """
478 cd is used to change remote working directory on server
495 cd is used to change remote working directory on server
479
496
480 Input:
497 Input:
481 remotefolder - current working directory
498 remotefolder - current working directory
482
499
483 Affects:
500 Affects:
484 self.remotefolder
501 self.remotefolder
485
502
486 Return:
503 Return:
487 0 in case of error else 1
504 0 in case of error else 1
488 """
505 """
489 if not self.status:
506 if not self.status:
490 return 0
507 return 0
491
508
492 if remotefolder == self.remotefolder:
509 if remotefolder == self.remotefolder:
493 return 1
510 return 1
494
511
512 chk_command = "cd %s; pwd" %remotefolder
513 mkdir_command = "mkdir %s" %remotefolder
514
515 if not self.__execute(chk_command):
516 if not self.__execute(mkdir_command):
517 self.remotefolder = None
518 return 0
519
495 self.remotefolder = remotefolder
520 self.remotefolder = remotefolder
496
521
497 return 1
522 return 1
498
523
499 def sendFile(self, fullfilename):
524 def sendFile(self, fullfilename):
500
525
501 if not self.status:
526 if not self.status:
502 return 0
527 return 0
503
528
504 try:
529 try:
505 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
530 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
506 except:
531 except:
507 return 0
532 return 0
508
533
509 return 1
534 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
535 command = 'chmod 775 %s' %remotefile
536
537 return self.__execute(command)
510
538
511 class SendToServer(ProcessingUnit):
539 class SendToServer(ProcessingUnit):
512
540
513 def __init__(self):
541 def __init__(self):
514
542
515 ProcessingUnit.__init__(self)
543 ProcessingUnit.__init__(self)
516
544
517 self.isConfig = False
545 self.isConfig = False
518 self.clientObj = None
546 self.clientObj = None
519
547
520 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp'):
548 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp'):
521
549
522 self.clientObj = None
550 self.clientObj = None
523 self.localfolder = localfolder
551 self.localfolder = localfolder
524 self.ext = ext
552 self.ext = ext
525 self.period = period
553 self.period = period
526
554
527 if str.lower(protocol) == 'ftp':
555 if str.lower(protocol) == 'ftp':
528 self.clientObj = FTPClient(server, username, password, remotefolder, period)
556 self.clientObj = FTPClient(server, username, password, remotefolder, period)
529
557
530 if str.lower(protocol) == 'ssh':
558 if str.lower(protocol) == 'ssh':
531 self.clientObj = SSHClient(server, username, password, remotefolder, period)
559 self.clientObj = SSHClient(server, username, password, remotefolder, period)
532
560
533 if not self.clientObj:
561 if not self.clientObj:
534 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
562 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
535
563
536 self.clientObj.start()
564 self.clientObj.start()
537
565
538 def findFiles(self):
566 def findFiles(self):
539
567
540 filenameList = glob.glob1(self.localfolder, '*%s' %self.ext)
568 filenameList = glob.glob1(self.localfolder, '*%s' %self.ext)
541
569
542 if len(filenameList) < 1:
570 if len(filenameList) < 1:
543 return []
571 return []
544
572
545 fullfilenameList = [os.path.join(self.localfolder, thisFile) for thisFile in filenameList]
573 fullfilenameList = [os.path.join(self.localfolder, thisFile) for thisFile in filenameList]
546
574
547 return fullfilenameList
575 return fullfilenameList
548
576
549 def run(self, **kwargs):
577 def run(self, **kwargs):
550
578
551 if not self.isConfig:
579 if not self.isConfig:
552 self.init = time.time()
580 self.init = time.time()
553 self.setup(**kwargs)
581 self.setup(**kwargs)
554 self.isConfig = True
582 self.isConfig = True
555
583
556 if time.time() - self.init >= self.period:
584 if time.time() - self.init >= self.period:
557 fullfilenameList = self.findFiles()
585 fullfilenameList = self.findFiles()
558 self.clientObj.updateFileList(fullfilenameList)
586 self.clientObj.updateFileList(fullfilenameList)
559 self.init = time.time()
587 self.init = time.time()
560
588
561 def close(self):
589 def close(self):
562 print "[Remote Server] Stopping thread"
590 print "[Remote Server] Stopping thread"
563 self.clientObj.stop()
591 self.clientObj.stop()
564
592
565
593
566 class FTP(object):
594 class FTP(object):
567 """
595 """
568 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
596 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
569
597
570 Non-standard Python modules used: None
598 Non-standard Python modules used: None
571
599
572 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
600 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
573 """
601 """
574
602
575 def __init__(self,server = None, username=None, password=None, remotefolder=None):
603 def __init__(self,server = None, username=None, password=None, remotefolder=None):
576 """
604 """
577 This method is used to setting parameters for FTP and establishing connection to remote server
605 This method is used to setting parameters for FTP and establishing connection to remote server
578
606
579 Inputs:
607 Inputs:
580 server - remote server IP Address
608 server - remote server IP Address
581
609
582 username - remote server Username
610 username - remote server Username
583
611
584 password - remote server password
612 password - remote server password
585
613
586 remotefolder - remote server current working directory
614 remotefolder - remote server current working directory
587
615
588 Return: void
616 Return: void
589
617
590 Affects:
618 Affects:
591 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
619 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
592
620
593 self.folderList - sub-folder list of remote folder
621 self.folderList - sub-folder list of remote folder
594
622
595 self.fileList - file list of remote folder
623 self.fileList - file list of remote folder
596
624
597
625
598 """
626 """
599
627
600 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
628 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
601 server, username, password, remotefolder = self.parmsByDefault()
629 server, username, password, remotefolder = self.parmsByDefault()
602
630
603 self.server = server
631 self.server = server
604 self.username = username
632 self.username = username
605 self.password = password
633 self.password = password
606 self.remotefolder = remotefolder
634 self.remotefolder = remotefolder
607 self.file = None
635 self.file = None
608 self.ftp = None
636 self.ftp = None
609 self.status = 0
637 self.status = 0
610
638
611 try:
639 try:
612 self.ftp = ftplib.FTP(self.server)
640 self.ftp = ftplib.FTP(self.server)
613 self.ftp.login(self.username,self.password)
641 self.ftp.login(self.username,self.password)
614 self.ftp.cwd(self.remotefolder)
642 self.ftp.cwd(self.remotefolder)
615 # print 'Connect to FTP Server: Successfully'
643 # print 'Connect to FTP Server: Successfully'
616
644
617 except ftplib.all_errors:
645 except ftplib.all_errors:
618 print 'Error FTP Service'
646 print 'Error FTP Service'
619 self.status = 1
647 self.status = 1
620 return
648 return
621
649
622
650
623
651
624 self.dirList = []
652 self.dirList = []
625
653
626 try:
654 try:
627 self.dirList = self.ftp.nlst()
655 self.dirList = self.ftp.nlst()
628
656
629 except ftplib.error_perm, resp:
657 except ftplib.error_perm, resp:
630 if str(resp) == "550 No files found":
658 if str(resp) == "550 No files found":
631 print "no files in this directory"
659 print "no files in this directory"
632 self.status = 1
660 self.status = 1
633 return
661 return
634
662
635 except ftplib.all_errors:
663 except ftplib.all_errors:
636 print 'Error Displaying Dir-Files'
664 print 'Error Displaying Dir-Files'
637 self.status = 1
665 self.status = 1
638 return
666 return
639
667
640 self.fileList = []
668 self.fileList = []
641 self.folderList = []
669 self.folderList = []
642 #only for test
670 #only for test
643 for f in self.dirList:
671 for f in self.dirList:
644 name, ext = os.path.splitext(f)
672 name, ext = os.path.splitext(f)
645 if ext != '':
673 if ext != '':
646 self.fileList.append(f)
674 self.fileList.append(f)
647 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
675 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
648
676
649 def parmsByDefault(self):
677 def parmsByDefault(self):
650 server = 'jro-app.igp.gob.pe'
678 server = 'jro-app.igp.gob.pe'
651 username = 'wmaster'
679 username = 'wmaster'
652 password = 'mst2010vhf'
680 password = 'mst2010vhf'
653 remotefolder = '/home/wmaster/graficos'
681 remotefolder = '/home/wmaster/graficos'
654
682
655 return server, username, password, remotefolder
683 return server, username, password, remotefolder
656
684
657
685
658 def mkd(self,dirname):
686 def mkd(self,dirname):
659 """
687 """
660 mkd is used to make directory in remote server
688 mkd is used to make directory in remote server
661
689
662 Input:
690 Input:
663 dirname - directory name
691 dirname - directory name
664
692
665 Return:
693 Return:
666 1 in error case else 0
694 1 in error case else 0
667 """
695 """
668 try:
696 try:
669 self.ftp.mkd(dirname)
697 self.ftp.mkd(dirname)
670 except:
698 except:
671 print 'Error creating remote folder:%s'%dirname
699 print 'Error creating remote folder:%s'%dirname
672 return 1
700 return 1
673
701
674 return 0
702 return 0
675
703
676
704
677 def delete(self,filename):
705 def delete(self,filename):
678 """
706 """
679 delete is used to delete file in current working directory of remote server
707 delete is used to delete file in current working directory of remote server
680
708
681 Input:
709 Input:
682 filename - filename to delete in remote folder
710 filename - filename to delete in remote folder
683
711
684 Return:
712 Return:
685 1 in error case else 0
713 1 in error case else 0
686 """
714 """
687
715
688 try:
716 try:
689 self.ftp.delete(filename)
717 self.ftp.delete(filename)
690 except:
718 except:
691 print 'Error deleting remote file:%s'%filename
719 print 'Error deleting remote file:%s'%filename
692 return 1
720 return 1
693
721
694 return 0
722 return 0
695
723
696 def download(self,filename,localfolder):
724 def download(self,filename,localfolder):
697 """
725 """
698 download is used to downloading file from remote folder into local folder
726 download is used to downloading file from remote folder into local folder
699
727
700 Inputs:
728 Inputs:
701 filename - filename to donwload
729 filename - filename to donwload
702
730
703 localfolder - directory local to store filename
731 localfolder - directory local to store filename
704
732
705 Returns:
733 Returns:
706 self.status - 1 in error case else 0
734 self.status - 1 in error case else 0
707 """
735 """
708
736
709 self.status = 0
737 self.status = 0
710
738
711
739
712 if not(filename in self.fileList):
740 if not(filename in self.fileList):
713 print 'filename:%s not exists'%filename
741 print 'filename:%s not exists'%filename
714 self.status = 1
742 self.status = 1
715 return self.status
743 return self.status
716
744
717 newfilename = os.path.join(localfolder,filename)
745 newfilename = os.path.join(localfolder,filename)
718
746
719 self.file = open(newfilename, 'wb')
747 self.file = open(newfilename, 'wb')
720
748
721 try:
749 try:
722 print 'Download: ' + filename
750 print 'Download: ' + filename
723 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
751 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
724 print 'Download Complete'
752 print 'Download Complete'
725 except ftplib.all_errors:
753 except ftplib.all_errors:
726 print 'Error Downloading ' + filename
754 print 'Error Downloading ' + filename
727 self.status = 1
755 self.status = 1
728 return self.status
756 return self.status
729
757
730 self.file.close()
758 self.file.close()
731
759
732 return self.status
760 return self.status
733
761
734
762
735 def __handleDownload(self,block):
763 def __handleDownload(self,block):
736 """
764 """
737 __handleDownload is used to handle writing file
765 __handleDownload is used to handle writing file
738 """
766 """
739 self.file.write(block)
767 self.file.write(block)
740
768
741
769
742 def upload(self,filename,remotefolder=None):
770 def upload(self,filename,remotefolder=None):
743 """
771 """
744 upload is used to uploading local file to remote directory
772 upload is used to uploading local file to remote directory
745
773
746 Inputs:
774 Inputs:
747 filename - full path name of local file to store in remote directory
775 filename - full path name of local file to store in remote directory
748
776
749 remotefolder - remote directory
777 remotefolder - remote directory
750
778
751 Returns:
779 Returns:
752 self.status - 1 in error case else 0
780 self.status - 1 in error case else 0
753 """
781 """
754
782
755 if remotefolder == None:
783 if remotefolder == None:
756 remotefolder = self.remotefolder
784 remotefolder = self.remotefolder
757
785
758 self.status = 0
786 self.status = 0
759
787
760 try:
788 try:
761 self.ftp.cwd(remotefolder)
789 self.ftp.cwd(remotefolder)
762
790
763 self.file = open(filename, 'rb')
791 self.file = open(filename, 'rb')
764
792
765 (head, tail) = os.path.split(filename)
793 (head, tail) = os.path.split(filename)
766
794
767 command = "STOR " + tail
795 command = "STOR " + tail
768
796
769 print 'Uploading: ' + tail
797 print 'Uploading: ' + tail
770 self.ftp.storbinary(command, self.file)
798 self.ftp.storbinary(command, self.file)
771 print 'Upload Completed'
799 print 'Upload Completed'
772
800
773 except ftplib.all_errors:
801 except ftplib.all_errors:
774 print 'Error Uploading ' + tail
802 print 'Error Uploading ' + tail
775 self.status = 1
803 self.status = 1
776 return self.status
804 return self.status
777
805
778 self.file.close()
806 self.file.close()
779
807
780 #back to initial directory in __init__()
808 #back to initial directory in __init__()
781 self.ftp.cwd(self.remotefolder)
809 self.ftp.cwd(self.remotefolder)
782
810
783 return self.status
811 return self.status
784
812
785
813
786 def dir(self,remotefolder):
814 def dir(self,remotefolder):
787 """
815 """
788 dir is used to change working directory of remote server and get folder and file list
816 dir is used to change working directory of remote server and get folder and file list
789
817
790 Input:
818 Input:
791 remotefolder - current working directory
819 remotefolder - current working directory
792
820
793 Affects:
821 Affects:
794 self.fileList - file list of working directory
822 self.fileList - file list of working directory
795
823
796 Return:
824 Return:
797 infoList - list with filenames and size of file in bytes
825 infoList - list with filenames and size of file in bytes
798
826
799 self.folderList - folder list
827 self.folderList - folder list
800 """
828 """
801
829
802 self.remotefolder = remotefolder
830 self.remotefolder = remotefolder
803 print 'Change to ' + self.remotefolder
831 print 'Change to ' + self.remotefolder
804 try:
832 try:
805 self.ftp.cwd(remotefolder)
833 self.ftp.cwd(remotefolder)
806 except ftplib.all_errors:
834 except ftplib.all_errors:
807 print 'Error Change to ' + self.remotefolder
835 print 'Error Change to ' + self.remotefolder
808 infoList = None
836 infoList = None
809 self.folderList = None
837 self.folderList = None
810 return infoList,self.folderList
838 return infoList,self.folderList
811
839
812 self.dirList = []
840 self.dirList = []
813
841
814 try:
842 try:
815 self.dirList = self.ftp.nlst()
843 self.dirList = self.ftp.nlst()
816
844
817 except ftplib.error_perm, resp:
845 except ftplib.error_perm, resp:
818 if str(resp) == "550 No files found":
846 if str(resp) == "550 No files found":
819 print "no files in this directory"
847 print "no files in this directory"
820 infoList = None
848 infoList = None
821 self.folderList = None
849 self.folderList = None
822 return infoList,self.folderList
850 return infoList,self.folderList
823 except ftplib.all_errors:
851 except ftplib.all_errors:
824 print 'Error Displaying Dir-Files'
852 print 'Error Displaying Dir-Files'
825 infoList = None
853 infoList = None
826 self.folderList = None
854 self.folderList = None
827 return infoList,self.folderList
855 return infoList,self.folderList
828
856
829 infoList = []
857 infoList = []
830 self.fileList = []
858 self.fileList = []
831 self.folderList = []
859 self.folderList = []
832 for f in self.dirList:
860 for f in self.dirList:
833 name,ext = os.path.splitext(f)
861 name,ext = os.path.splitext(f)
834 if ext != '':
862 if ext != '':
835 self.fileList.append(f)
863 self.fileList.append(f)
836 value = (f,self.ftp.size(f))
864 value = (f,self.ftp.size(f))
837 infoList.append(value)
865 infoList.append(value)
838
866
839 if ext == '':
867 if ext == '':
840 self.folderList.append(f)
868 self.folderList.append(f)
841
869
842 return infoList,self.folderList
870 return infoList,self.folderList
843
871
844
872
845 def close(self):
873 def close(self):
846 """
874 """
847 close is used to close and end FTP connection
875 close is used to close and end FTP connection
848
876
849 Inputs: None
877 Inputs: None
850
878
851 Return: void
879 Return: void
852
880
853 """
881 """
854 self.ftp.close()
882 self.ftp.close()
855
883
856 class SendByFTP(Operation):
884 class SendByFTP(Operation):
857
885
858 def __init__(self):
886 def __init__(self):
859
887
860 self.status = 1
888 self.status = 1
861 self.counter = 0
889 self.counter = 0
862
890
863 def error_print(self, ValueError):
891 def error_print(self, ValueError):
864
892
865 print ValueError, 'Error FTP'
893 print ValueError, 'Error FTP'
866 print "don't worry the program is running..."
894 print "don't worry the program is running..."
867
895
868 def worker_ftp(self, server, username, password, remotefolder, filenameList):
896 def worker_ftp(self, server, username, password, remotefolder, filenameList):
869
897
870 self.ftpClientObj = FTP(server, username, password, remotefolder)
898 self.ftpClientObj = FTP(server, username, password, remotefolder)
871 for filename in filenameList:
899 for filename in filenameList:
872 self.ftpClientObj.upload(filename)
900 self.ftpClientObj.upload(filename)
873 self.ftpClientObj.close()
901 self.ftpClientObj.close()
874
902
875 def ftp_thread(self, server, username, password, remotefolder):
903 def ftp_thread(self, server, username, password, remotefolder):
876 if not(self.status):
904 if not(self.status):
877 return
905 return
878
906
879 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
907 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
880 p.start()
908 p.start()
881
909
882 p.join(3)
910 p.join(3)
883
911
884 if p.is_alive():
912 if p.is_alive():
885 p.terminate()
913 p.terminate()
886 p.join()
914 p.join()
887 print 'killing ftp process...'
915 print 'killing ftp process...'
888 self.status = 0
916 self.status = 0
889 return
917 return
890
918
891 self.status = 1
919 self.status = 1
892 return
920 return
893
921
894 def filterByExt(self, ext, localfolder):
922 def filterByExt(self, ext, localfolder):
895 fnameList = glob.glob1(localfolder,ext)
923 fnameList = glob.glob1(localfolder,ext)
896 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
924 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
897
925
898 if len(self.filenameList) == 0:
926 if len(self.filenameList) == 0:
899 self.status = 0
927 self.status = 0
900
928
901 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
929 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
902
930
903 self.counter += 1
931 self.counter += 1
904 if self.counter >= period:
932 if self.counter >= period:
905 self.filterByExt(ext, localfolder)
933 self.filterByExt(ext, localfolder)
906
934
907 self.ftp_thread(server, username, password, remotefolder)
935 self.ftp_thread(server, username, password, remotefolder)
908
936
909 self.counter = 0
937 self.counter = 0
910
938
911 self.status = 1
939 self.status = 1
912
940
General Comments 0
You need to be logged in to leave comments. Login now