##// END OF EJS Templates
schain
RSS
Bug fixed: FTP Server thread stopped when an error sending file occurred
Miguel Valdez -
r646:8add51ecf562
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,988 +1,982
1 '''
1 '''
2 @author: Daniel Suarez
2 @author: Daniel Suarez
3 '''
3 '''
4 import os
4 import os
5 import glob
5 import glob
6 import ftplib
6 import ftplib
7
7
8 try:
8 try:
9 import paramiko
9 import paramiko
10 import scp
10 import scp
11 except:
11 except:
12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
13
13
14 import time
14 import time
15
15
16 import threading
16 import threading
17 Thread = threading.Thread
17 Thread = threading.Thread
18
18
19 # try:
19 # try:
20 # from gevent import sleep
20 # from gevent import sleep
21 # except:
21 # except:
22 from time import sleep
22 from time import sleep
23
23
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25
25
26 class Remote(Thread):
26 class Remote(Thread):
27 """
27 """
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 used to upload or download files remotely.
29 used to upload or download files remotely.
30
30
31 Non-standard Python modules used:
31 Non-standard Python modules used:
32 None
32 None
33
33
34 Written by:
34 Written by:
35
35
36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
37
37
38 """
38 """
39
39
40 server = None
40 server = None
41 username = None
41 username = None
42 password = None
42 password = None
43 remotefolder = None
43 remotefolder = None
44
44
45 period = 60
45 period = 60
46 fileList = []
46 fileList = []
47 bussy = False
47 bussy = False
48
48
49 def __init__(self, server, username, password, remotefolder, period=60):
49 def __init__(self, server, username, password, remotefolder, period=60):
50
50
51 Thread.__init__(self)
51 Thread.__init__(self)
52
52
53 self.setDaemon(True)
53 self.setDaemon(True)
54
54
55 self.status = 0
55 self.status = 0
56
56
57 self.period = period
57 self.period = period
58 self.fileList = []
58 self.fileList = []
59 self.bussy = False
59 self.bussy = False
60
60
61 self.stopFlag = False
61 self.stopFlag = False
62
62
63 print "[Remote Server] Opening server: %s" %server
63 print "[Remote Server] Opening server: %s" %server
64 if self.open(server, username, password, remotefolder):
64 if self.open(server, username, password, remotefolder):
65 print "[Remote Server] %s server was opened successfully" %server
65 print "[Remote Server] %s server was opened successfully" %server
66
66
67 self.mutex = threading.Lock()
67 self.mutex = threading.Lock()
68
68
69 def stop(self):
69 def stop(self):
70
70
71 self.stopFlag = True
71 self.stopFlag = True
72 self.join(10)
72 self.join(10)
73
73
74 def open(self):
74 def open(self):
75 """
75 """
76 Connect to server and create a connection class (FTP or SSH) to remote server.
76 Connect to server and create a connection class (FTP or SSH) to remote server.
77 """
77 """
78 raise NotImplementedError, "Implement this method in child class"
78 raise NotImplementedError, "Implement this method in child class"
79
79
80 def close(self):
80 def close(self):
81 """
81 """
82 Close connection to server
82 Close connection to server
83 """
83 """
84 raise NotImplementedError, "Implement this method in child class"
84 raise NotImplementedError, "Implement this method in child class"
85
85
86 def mkdir(self, remotefolder):
86 def mkdir(self, remotefolder):
87 """
87 """
88 Create a folder remotely
88 Create a folder remotely
89 """
89 """
90 raise NotImplementedError, "Implement this method in child class"
90 raise NotImplementedError, "Implement this method in child class"
91
91
92 def cd(self, remotefolder):
92 def cd(self, remotefolder):
93 """
93 """
94 Change working directory in remote server
94 Change working directory in remote server
95 """
95 """
96 raise NotImplementedError, "Implement this method in child class"
96 raise NotImplementedError, "Implement this method in child class"
97
97
98 def download(self, filename, localfolder=None):
98 def download(self, filename, localfolder=None):
99 """
99 """
100 Download a file from server to local host
100 Download a file from server to local host
101 """
101 """
102 raise NotImplementedError, "Implement this method in child class"
102 raise NotImplementedError, "Implement this method in child class"
103
103
104 def sendFile(self, fullfilename):
104 def sendFile(self, fullfilename):
105 """
105 """
106 sendFile method is used to upload a local file to the current directory in remote server
106 sendFile method is used to upload a local file to the current directory in remote server
107
107
108 Inputs:
108 Inputs:
109 fullfilename - full path name of local file to store in remote directory
109 fullfilename - full path name of local file to store in remote directory
110
110
111 Returns:
111 Returns:
112 0 in error case else 1
112 0 in error case else 1
113 """
113 """
114 raise NotImplementedError, "Implement this method in child class"
114 raise NotImplementedError, "Implement this method in child class"
115
115
116 def upload(self, fullfilename, remotefolder=None):
116 def upload(self, fullfilename, remotefolder=None):
117 """
117 """
118 upload method is used to upload a local file to remote directory. This method changes
118 upload method is used to upload a local file to remote directory. This method changes
119 working directory before sending a file.
119 working directory before sending a file.
120
120
121 Inputs:
121 Inputs:
122 fullfilename - full path name of local file to store in remote directory
122 fullfilename - full path name of local file to store in remote directory
123
123
124 remotefolder - remote directory
124 remotefolder - remote directory
125
125
126 Returns:
126 Returns:
127 0 in error case else 1
127 0 in error case else 1
128 """
128 """
129 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
129 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
130
130
131 if not self.status:
131 if not self.status:
132 return 0
132 return 0
133
133
134 if remotefolder == None:
134 if remotefolder == None:
135 remotefolder = self.remotefolder
135 remotefolder = self.remotefolder
136
136
137 if not self.cd(remotefolder):
137 if not self.cd(remotefolder):
138 return 0
138 return 0
139
139
140 if not self.sendFile(fullfilename):
140 if not self.sendFile(fullfilename):
141 print "[Remote Server] Error uploading file %s" %fullfilename
141 print "[Remote Server] Error uploading file %s" %fullfilename
142 return 0
142 return 0
143
143
144 print "[Remote Server] upload finished successfully"
144 print "[Remote Server] upload finished successfully"
145
145
146 return 1
146 return 1
147
147
148 def delete(self, filename):
148 def delete(self, filename):
149 """
149 """
150 Remove a file from remote server
150 Remove a file from remote server
151 """
151 """
152 pass
152 pass
153
153
154 def updateFileList(self, fileList):
154 def updateFileList(self, fileList):
155 """
155 """
156 Remove a file from remote server
156 Remove a file from remote server
157 """
157 """
158
158
159 if fileList == self.fileList:
159 if fileList == self.fileList:
160 return 0
160 return 0
161
161
162 self.mutex.acquire()
162 self.mutex.acquire()
163 # init = time.time()
163 # init = time.time()
164 #
164 #
165 # while(self.bussy):
165 # while(self.bussy):
166 # sleep(0.1)
166 # sleep(0.1)
167 # if time.time() - init > 2*self.period:
167 # if time.time() - init > 2*self.period:
168 # return 0
168 # return 0
169
169
170 self.fileList = fileList
170 self.fileList = fileList
171 self.mutex.release()
171 self.mutex.release()
172 return 1
172 return 1
173
173
174 def run(self):
174 def run(self):
175
175
176 if not self.status:
176 if not self.status:
177 print "Finishing FTP service"
177 print "Finishing FTP service"
178 return
178 return
179
179
180 if not self.cd(self.remotefolder):
180 if not self.cd(self.remotefolder):
181 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
181 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
182
182
183 sts = True
184
185 while True:
183 while True:
186
184
187 for i in range(self.period):
185 for i in range(self.period):
188 if self.stopFlag:
186 if self.stopFlag:
189 break
187 break
190 sleep(1)
188 sleep(1)
191
189
192 if self.stopFlag:
190 if self.stopFlag:
193 break
191 break
194
192
195 # self.bussy = True
193 # self.bussy = True
196 self.mutex.acquire()
194 self.mutex.acquire()
197
195
198 for thisFile in self.fileList:
196 for thisFile in self.fileList:
199 sts = self.upload(thisFile, self.remotefolder)
197 self.upload(thisFile, self.remotefolder)
200 if not sts: break
201
198
202 self.mutex.release()
199 self.mutex.release()
203 # self.bussy = False
200 # self.bussy = False
204
201
205 if not sts:
206 break
207
208 print "[Remote Server] Thread stopped successfully"
202 print "[Remote Server] Thread stopped successfully"
209
203
210 class FTPClient(Remote):
204 class FTPClient(Remote):
211
205
212 __ftpClientObj = None
206 __ftpClientObj = None
213
207
214 def __init__(self, server, username, password, remotefolder, period=60):
208 def __init__(self, server, username, password, remotefolder, period=60):
215 """
209 """
216 """
210 """
217 Remote.__init__(self, server, username, password, remotefolder, period)
211 Remote.__init__(self, server, username, password, remotefolder, period)
218
212
219 def open(self, server, username, password, remotefolder):
213 def open(self, server, username, password, remotefolder):
220
214
221 """
215 """
222 This method is used to set FTP parameters and establish a connection to remote server
216 This method is used to set FTP parameters and establish a connection to remote server
223
217
224 Inputs:
218 Inputs:
225 server - remote server IP Address
219 server - remote server IP Address
226
220
227 username - remote server Username
221 username - remote server Username
228
222
229 password - remote server password
223 password - remote server password
230
224
231 remotefolder - remote server current working directory
225 remotefolder - remote server current working directory
232
226
233 Return: void
227 Return: void
234
228
235 Affects:
229 Affects:
236 self.status - in case of error or fail connection this parameter is set to 0 else 1
230 self.status - in case of error or fail connection this parameter is set to 0 else 1
237
231
238 """
232 """
239
233
240 if server == None:
234 if server == None:
241 raise ValueError, "FTP server should be defined"
235 raise ValueError, "FTP server should be defined"
242
236
243 if username == None:
237 if username == None:
244 raise ValueError, "FTP username should be defined"
238 raise ValueError, "FTP username should be defined"
245
239
246 if password == None:
240 if password == None:
247 raise ValueError, "FTP password should be defined"
241 raise ValueError, "FTP password should be defined"
248
242
249 if remotefolder == None:
243 if remotefolder == None:
250 raise ValueError, "FTP remote folder should be defined"
244 raise ValueError, "FTP remote folder should be defined"
251
245
252 try:
246 try:
253 ftpClientObj = ftplib.FTP(server)
247 ftpClientObj = ftplib.FTP(server)
254 except ftplib.all_errors:
248 except ftplib.all_errors:
255 print "FTP server connection fail: %s" %server
249 print "FTP server connection fail: %s" %server
256 self.status = 0
250 self.status = 0
257 return 0
251 return 0
258
252
259 try:
253 try:
260 ftpClientObj.login(username, password)
254 ftpClientObj.login(username, password)
261 except ftplib.all_errors:
255 except ftplib.all_errors:
262 print "FTP username or password are incorrect"
256 print "FTP username or password are incorrect"
263 self.status = 0
257 self.status = 0
264 return 0
258 return 0
265
259
266 if remotefolder == None:
260 if remotefolder == None:
267 remotefolder = ftpClientObj.pwd()
261 remotefolder = ftpClientObj.pwd()
268 else:
262 else:
269 try:
263 try:
270 ftpClientObj.cwd(remotefolder)
264 ftpClientObj.cwd(remotefolder)
271 except ftplib.all_errors:
265 except ftplib.all_errors:
272 print "FTP remote folder is invalid: %s" %remotefolder
266 print "FTP remote folder is invalid: %s" %remotefolder
273 remotefolder = ftpClientObj.pwd()
267 remotefolder = ftpClientObj.pwd()
274
268
275 self.server = server
269 self.server = server
276 self.username = username
270 self.username = username
277 self.password = password
271 self.password = password
278 self.remotefolder = remotefolder
272 self.remotefolder = remotefolder
279 self.__ftpClientObj = ftpClientObj
273 self.__ftpClientObj = ftpClientObj
280 self.status = 1
274 self.status = 1
281
275
282 return 1
276 return 1
283
277
284 def close(self):
278 def close(self):
285 """
279 """
286 Close connection to remote server
280 Close connection to remote server
287 """
281 """
288 if not self.status:
282 if not self.status:
289 return 0
283 return 0
290
284
291 self.__ftpClientObj.close()
285 self.__ftpClientObj.close()
292
286
293 def mkdir(self, remotefolder):
287 def mkdir(self, remotefolder):
294 """
288 """
295 mkdir is used to make a new directory in remote server
289 mkdir is used to make a new directory in remote server
296
290
297 Input:
291 Input:
298 remotefolder - directory name
292 remotefolder - directory name
299
293
300 Return:
294 Return:
301 0 in error case else 1
295 0 in error case else 1
302 """
296 """
303 if not self.status:
297 if not self.status:
304 return 0
298 return 0
305
299
306 try:
300 try:
307 self.__ftpClientObj.mkd(dirname)
301 self.__ftpClientObj.mkd(dirname)
308 except ftplib.all_errors:
302 except ftplib.all_errors:
309 print "Error creating remote folder: %s" %remotefolder
303 print "Error creating remote folder: %s" %remotefolder
310 return 0
304 return 0
311
305
312 return 1
306 return 1
313
307
314 def cd(self, remotefolder):
308 def cd(self, remotefolder):
315 """
309 """
316 cd is used to change remote working directory on server
310 cd is used to change remote working directory on server
317
311
318 Input:
312 Input:
319 remotefolder - current working directory
313 remotefolder - current working directory
320
314
321 Affects:
315 Affects:
322 self.remotefolder
316 self.remotefolder
323
317
324 Return:
318 Return:
325 0 in case of error else 1
319 0 in case of error else 1
326 """
320 """
327 if not self.status:
321 if not self.status:
328 return 0
322 return 0
329
323
330 if remotefolder == self.remotefolder:
324 if remotefolder == self.remotefolder:
331 return 1
325 return 1
332
326
333 try:
327 try:
334 self.__ftpClientObj.cwd(remotefolder)
328 self.__ftpClientObj.cwd(remotefolder)
335 except ftplib.all_errors:
329 except ftplib.all_errors:
336 print 'Error changing to %s' %remotefolder
330 print 'Error changing to %s' %remotefolder
337 print 'Trying to create remote folder'
331 print 'Trying to create remote folder'
338
332
339 if not self.mkdir(remotefolder):
333 if not self.mkdir(remotefolder):
340 print 'Remote folder could not be created'
334 print 'Remote folder could not be created'
341 return 0
335 return 0
342
336
343 try:
337 try:
344 self.__ftpClientObj.cwd(remotefolder)
338 self.__ftpClientObj.cwd(remotefolder)
345 except ftplib.all_errors:
339 except ftplib.all_errors:
346 return 0
340 return 0
347
341
348 self.remotefolder = remotefolder
342 self.remotefolder = remotefolder
349
343
350 return 1
344 return 1
351
345
352 def sendFile(self, fullfilename):
346 def sendFile(self, fullfilename):
353
347
354 if not self.status:
348 if not self.status:
355 return 0
349 return 0
356
350
357 file = open(fullfilename, 'rb')
351 file = open(fullfilename, 'rb')
358
352
359 filename = os.path.basename(fullfilename)
353 filename = os.path.basename(fullfilename)
360
354
361 command = "STOR %s" %filename
355 command = "STOR %s" %filename
362
356
363 try:
357 try:
364 self.__ftpClientObj.storbinary(command, file)
358 self.__ftpClientObj.storbinary(command, file)
365 except ftplib.all_errors:
359 except ftplib.all_errors:
366 return 0
360 return 0
367
361
368 try:
362 try:
369 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
363 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
370 except ftplib.all_errors, e:
364 except ftplib.all_errors, e:
371 print e
365 print e
372
366
373 file.close()
367 file.close()
374
368
375 return 1
369 return 1
376
370
377 class SSHClient(Remote):
371 class SSHClient(Remote):
378
372
379 __sshClientObj = None
373 __sshClientObj = None
380 __scpClientObj = None
374 __scpClientObj = None
381
375
382 def __init__(self, server, username, password, remotefolder, period=60):
376 def __init__(self, server, username, password, remotefolder, period=60):
383 """
377 """
384 """
378 """
385 Remote.__init__(self, server, username, password, remotefolder, period)
379 Remote.__init__(self, server, username, password, remotefolder, period)
386
380
387 def open(self, server, username, password, remotefolder, port=22):
381 def open(self, server, username, password, remotefolder, port=22):
388
382
389 """
383 """
390 This method is used to set SSH parameters and establish a connection to a remote server
384 This method is used to set SSH parameters and establish a connection to a remote server
391
385
392 Inputs:
386 Inputs:
393 server - remote server IP Address
387 server - remote server IP Address
394
388
395 username - remote server Username
389 username - remote server Username
396
390
397 password - remote server password
391 password - remote server password
398
392
399 remotefolder - remote server current working directory
393 remotefolder - remote server current working directory
400
394
401 Return: void
395 Return: void
402
396
403 Affects:
397 Affects:
404 self.status - in case of error or fail connection this parameter is set to 0 else 1
398 self.status - in case of error or fail connection this parameter is set to 0 else 1
405
399
406 """
400 """
407
401
408 if server == None:
402 if server == None:
409 raise ValueError, "SSH server should be defined"
403 raise ValueError, "SSH server should be defined"
410
404
411 if username == None:
405 if username == None:
412 raise ValueError, "SSH username should be defined"
406 raise ValueError, "SSH username should be defined"
413
407
414 if password == None:
408 if password == None:
415 raise ValueError, "SSH password should be defined"
409 raise ValueError, "SSH password should be defined"
416
410
417 if remotefolder == None:
411 if remotefolder == None:
418 raise ValueError, "SSH remote folder should be defined"
412 raise ValueError, "SSH remote folder should be defined"
419
413
420 try:
414 try:
421 sshClientObj = paramiko.SSHClient()
415 sshClientObj = paramiko.SSHClient()
422 except:
416 except:
423 print "SSH server connection fail: %s" %server
417 print "SSH server connection fail: %s" %server
424 self.status = 0
418 self.status = 0
425 return 0
419 return 0
426
420
427 sshClientObj.load_system_host_keys()
421 sshClientObj.load_system_host_keys()
428 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
422 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
429
423
430 try:
424 try:
431 sshClientObj.connect(server, username=username, password=password, port=port)
425 sshClientObj.connect(server, username=username, password=password, port=port)
432 except :
426 except :
433 print "SSH username or password are incorrect: %s"
427 print "SSH username or password are incorrect: %s"
434 self.status = 0
428 self.status = 0
435 return 0
429 return 0
436
430
437 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
431 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
438
432
439 if remotefolder == None:
433 if remotefolder == None:
440 remotefolder = self.pwd()
434 remotefolder = self.pwd()
441
435
442 self.server = server
436 self.server = server
443 self.username = username
437 self.username = username
444 self.password = password
438 self.password = password
445 self.__sshClientObj = sshClientObj
439 self.__sshClientObj = sshClientObj
446 self.__scpClientObj = scpClientObj
440 self.__scpClientObj = scpClientObj
447 self.status = 1
441 self.status = 1
448
442
449 if not self.cd(remotefolder):
443 if not self.cd(remotefolder):
450 raise ValueError, "Could not access to remote folder: %s" %remotefolder
444 raise ValueError, "Could not access to remote folder: %s" %remotefolder
451 return 0
445 return 0
452
446
453 self.remotefolder = remotefolder
447 self.remotefolder = remotefolder
454
448
455 return 1
449 return 1
456
450
457 def close(self):
451 def close(self):
458 """
452 """
459 Close connection to remote server
453 Close connection to remote server
460 """
454 """
461 if not self.status:
455 if not self.status:
462 return 0
456 return 0
463
457
464 self.__sshObj.close()
458 self.__sshObj.close()
465
459
466 def __execute(self, command):
460 def __execute(self, command):
467 """
461 """
468 __execute a command on remote server
462 __execute a command on remote server
469
463
470 Input:
464 Input:
471 command - Exmaple 'ls -l'
465 command - Exmaple 'ls -l'
472
466
473 Return:
467 Return:
474 0 in error case else 1
468 0 in error case else 1
475 """
469 """
476 if not self.status:
470 if not self.status:
477 return 0
471 return 0
478
472
479 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
473 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
480
474
481 result = stderr.readlines()
475 result = stderr.readlines()
482 if len(result) > 1:
476 if len(result) > 1:
483 return 0
477 return 0
484
478
485 result = stdout.readlines()
479 result = stdout.readlines()
486 if len(result) > 1:
480 if len(result) > 1:
487 return result[0][:-1]
481 return result[0][:-1]
488
482
489 return 1
483 return 1
490
484
491 def mkdir(self, remotefolder):
485 def mkdir(self, remotefolder):
492 """
486 """
493 mkdir is used to make a new directory in remote server
487 mkdir is used to make a new directory in remote server
494
488
495 Input:
489 Input:
496 remotefolder - directory name
490 remotefolder - directory name
497
491
498 Return:
492 Return:
499 0 in error case else 1
493 0 in error case else 1
500 """
494 """
501
495
502 command = 'mkdir %s' %remotefolder
496 command = 'mkdir %s' %remotefolder
503
497
504 return self.__execute(command)
498 return self.__execute(command)
505
499
506 def pwd(self):
500 def pwd(self):
507
501
508 command = 'pwd'
502 command = 'pwd'
509
503
510 return self.__execute(command)
504 return self.__execute(command)
511
505
512 def cd(self, remotefolder):
506 def cd(self, remotefolder):
513 """
507 """
514 cd is used to change remote working directory on server
508 cd is used to change remote working directory on server
515
509
516 Input:
510 Input:
517 remotefolder - current working directory
511 remotefolder - current working directory
518
512
519 Affects:
513 Affects:
520 self.remotefolder
514 self.remotefolder
521
515
522 Return:
516 Return:
523 0 in case of error else 1
517 0 in case of error else 1
524 """
518 """
525 if not self.status:
519 if not self.status:
526 return 0
520 return 0
527
521
528 if remotefolder == self.remotefolder:
522 if remotefolder == self.remotefolder:
529 return 1
523 return 1
530
524
531 chk_command = "cd %s; pwd" %remotefolder
525 chk_command = "cd %s; pwd" %remotefolder
532 mkdir_command = "mkdir %s" %remotefolder
526 mkdir_command = "mkdir %s" %remotefolder
533
527
534 if not self.__execute(chk_command):
528 if not self.__execute(chk_command):
535 if not self.__execute(mkdir_command):
529 if not self.__execute(mkdir_command):
536 self.remotefolder = None
530 self.remotefolder = None
537 return 0
531 return 0
538
532
539 self.remotefolder = remotefolder
533 self.remotefolder = remotefolder
540
534
541 return 1
535 return 1
542
536
543 def sendFile(self, fullfilename):
537 def sendFile(self, fullfilename):
544
538
545 if not self.status:
539 if not self.status:
546 return 0
540 return 0
547
541
548 try:
542 try:
549 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
543 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
550 except:
544 except:
551 return 0
545 return 0
552
546
553 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
547 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
554 command = 'chmod 775 %s' %remotefile
548 command = 'chmod 775 %s' %remotefile
555
549
556 return self.__execute(command)
550 return self.__execute(command)
557
551
558 class SendToServer(ProcessingUnit):
552 class SendToServer(ProcessingUnit):
559
553
560 def __init__(self):
554 def __init__(self):
561
555
562 ProcessingUnit.__init__(self)
556 ProcessingUnit.__init__(self)
563
557
564 self.isConfig = False
558 self.isConfig = False
565 self.clientObj = None
559 self.clientObj = None
566
560
567 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
561 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
568
562
569 self.clientObj = None
563 self.clientObj = None
570 self.localfolder = localfolder
564 self.localfolder = localfolder
571 self.ext = ext
565 self.ext = ext
572 self.period = period
566 self.period = period
573
567
574 if str.lower(protocol) == 'ftp':
568 if str.lower(protocol) == 'ftp':
575 self.clientObj = FTPClient(server, username, password, remotefolder, period)
569 self.clientObj = FTPClient(server, username, password, remotefolder, period)
576
570
577 if str.lower(protocol) == 'ssh':
571 if str.lower(protocol) == 'ssh':
578 self.clientObj = SSHClient(server, username, password, remotefolder, period)
572 self.clientObj = SSHClient(server, username, password, remotefolder, period)
579
573
580 if not self.clientObj:
574 if not self.clientObj:
581 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
575 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
582
576
583 self.clientObj.start()
577 self.clientObj.start()
584
578
585 def findFiles(self):
579 def findFiles(self):
586
580
587 if not type(self.localfolder) == list:
581 if not type(self.localfolder) == list:
588 folderList = [self.localfolder]
582 folderList = [self.localfolder]
589 else:
583 else:
590 folderList = self.localfolder
584 folderList = self.localfolder
591
585
592 #Remove duplicate items
586 #Remove duplicate items
593 folderList = list(set(folderList))
587 folderList = list(set(folderList))
594
588
595 fullfilenameList = []
589 fullfilenameList = []
596
590
597 for thisFolder in folderList:
591 for thisFolder in folderList:
598
592
599 print "[Remote Server]: Searching files on %s" %thisFolder
593 print "[Remote Server]: Searching files on %s" %thisFolder
600
594
601 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
595 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
602
596
603 if len(filenameList) < 1:
597 if len(filenameList) < 1:
604 continue
598 continue
605
599
606 for thisFile in filenameList:
600 for thisFile in filenameList:
607 fullfilename = os.path.join(thisFolder, thisFile)
601 fullfilename = os.path.join(thisFolder, thisFile)
608
602
609 if fullfilename in fullfilenameList:
603 if fullfilename in fullfilenameList:
610 continue
604 continue
611
605
612 #Only files modified in the last 30 minutes are considered
606 #Only files modified in the last 30 minutes are considered
613 if os.path.getmtime(fullfilename) < time.time() - 30*60:
607 if os.path.getmtime(fullfilename) < time.time() - 30*60:
614 continue
608 continue
615
609
616 fullfilenameList.append(fullfilename)
610 fullfilenameList.append(fullfilename)
617
611
618 return fullfilenameList
612 return fullfilenameList
619
613
620 def run(self, **kwargs):
614 def run(self, **kwargs):
621
615
622 if not self.isConfig:
616 if not self.isConfig:
623 self.init = time.time()
617 self.init = time.time()
624 self.setup(**kwargs)
618 self.setup(**kwargs)
625 self.isConfig = True
619 self.isConfig = True
626
620
627 if time.time() - self.init >= self.period:
621 if time.time() - self.init >= self.period:
628 fullfilenameList = self.findFiles()
622 fullfilenameList = self.findFiles()
629
623
630 if self.clientObj.updateFileList(fullfilenameList):
624 if self.clientObj.updateFileList(fullfilenameList):
631 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
625 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
632
626
633 self.init = time.time()
627 self.init = time.time()
634
628
635 def close(self):
629 def close(self):
636 print "[Remote Server] Stopping thread"
630 print "[Remote Server] Stopping thread"
637 self.clientObj.stop()
631 self.clientObj.stop()
638
632
639
633
640 class FTP(object):
634 class FTP(object):
641 """
635 """
642 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
636 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
643
637
644 Non-standard Python modules used: None
638 Non-standard Python modules used: None
645
639
646 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
640 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
647 """
641 """
648
642
649 def __init__(self,server = None, username=None, password=None, remotefolder=None):
643 def __init__(self,server = None, username=None, password=None, remotefolder=None):
650 """
644 """
651 This method is used to setting parameters for FTP and establishing connection to remote server
645 This method is used to setting parameters for FTP and establishing connection to remote server
652
646
653 Inputs:
647 Inputs:
654 server - remote server IP Address
648 server - remote server IP Address
655
649
656 username - remote server Username
650 username - remote server Username
657
651
658 password - remote server password
652 password - remote server password
659
653
660 remotefolder - remote server current working directory
654 remotefolder - remote server current working directory
661
655
662 Return: void
656 Return: void
663
657
664 Affects:
658 Affects:
665 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
659 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
666
660
667 self.folderList - sub-folder list of remote folder
661 self.folderList - sub-folder list of remote folder
668
662
669 self.fileList - file list of remote folder
663 self.fileList - file list of remote folder
670
664
671
665
672 """
666 """
673
667
674 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
668 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
675 server, username, password, remotefolder = self.parmsByDefault()
669 server, username, password, remotefolder = self.parmsByDefault()
676
670
677 self.server = server
671 self.server = server
678 self.username = username
672 self.username = username
679 self.password = password
673 self.password = password
680 self.remotefolder = remotefolder
674 self.remotefolder = remotefolder
681 self.file = None
675 self.file = None
682 self.ftp = None
676 self.ftp = None
683 self.status = 0
677 self.status = 0
684
678
685 try:
679 try:
686 self.ftp = ftplib.FTP(self.server)
680 self.ftp = ftplib.FTP(self.server)
687 self.ftp.login(self.username,self.password)
681 self.ftp.login(self.username,self.password)
688 self.ftp.cwd(self.remotefolder)
682 self.ftp.cwd(self.remotefolder)
689 # print 'Connect to FTP Server: Successfully'
683 # print 'Connect to FTP Server: Successfully'
690
684
691 except ftplib.all_errors:
685 except ftplib.all_errors:
692 print 'Error FTP Service'
686 print 'Error FTP Service'
693 self.status = 1
687 self.status = 1
694 return
688 return
695
689
696
690
697
691
698 self.dirList = []
692 self.dirList = []
699
693
700 try:
694 try:
701 self.dirList = self.ftp.nlst()
695 self.dirList = self.ftp.nlst()
702
696
703 except ftplib.error_perm, resp:
697 except ftplib.error_perm, resp:
704 if str(resp) == "550 No files found":
698 if str(resp) == "550 No files found":
705 print "no files in this directory"
699 print "no files in this directory"
706 self.status = 1
700 self.status = 1
707 return
701 return
708
702
709 except ftplib.all_errors:
703 except ftplib.all_errors:
710 print 'Error Displaying Dir-Files'
704 print 'Error Displaying Dir-Files'
711 self.status = 1
705 self.status = 1
712 return
706 return
713
707
714 self.fileList = []
708 self.fileList = []
715 self.folderList = []
709 self.folderList = []
716 #only for test
710 #only for test
717 for f in self.dirList:
711 for f in self.dirList:
718 name, ext = os.path.splitext(f)
712 name, ext = os.path.splitext(f)
719 if ext != '':
713 if ext != '':
720 self.fileList.append(f)
714 self.fileList.append(f)
721 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
715 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
722
716
723 def parmsByDefault(self):
717 def parmsByDefault(self):
724 server = 'jro-app.igp.gob.pe'
718 server = 'jro-app.igp.gob.pe'
725 username = 'wmaster'
719 username = 'wmaster'
726 password = 'mst2010vhf'
720 password = 'mst2010vhf'
727 remotefolder = '/home/wmaster/graficos'
721 remotefolder = '/home/wmaster/graficos'
728
722
729 return server, username, password, remotefolder
723 return server, username, password, remotefolder
730
724
731
725
732 def mkd(self,dirname):
726 def mkd(self,dirname):
733 """
727 """
734 mkd is used to make directory in remote server
728 mkd is used to make directory in remote server
735
729
736 Input:
730 Input:
737 dirname - directory name
731 dirname - directory name
738
732
739 Return:
733 Return:
740 1 in error case else 0
734 1 in error case else 0
741 """
735 """
742 try:
736 try:
743 self.ftp.mkd(dirname)
737 self.ftp.mkd(dirname)
744 except:
738 except:
745 print 'Error creating remote folder:%s'%dirname
739 print 'Error creating remote folder:%s'%dirname
746 return 1
740 return 1
747
741
748 return 0
742 return 0
749
743
750
744
751 def delete(self,filename):
745 def delete(self,filename):
752 """
746 """
753 delete is used to delete file in current working directory of remote server
747 delete is used to delete file in current working directory of remote server
754
748
755 Input:
749 Input:
756 filename - filename to delete in remote folder
750 filename - filename to delete in remote folder
757
751
758 Return:
752 Return:
759 1 in error case else 0
753 1 in error case else 0
760 """
754 """
761
755
762 try:
756 try:
763 self.ftp.delete(filename)
757 self.ftp.delete(filename)
764 except:
758 except:
765 print 'Error deleting remote file:%s'%filename
759 print 'Error deleting remote file:%s'%filename
766 return 1
760 return 1
767
761
768 return 0
762 return 0
769
763
770 def download(self,filename,localfolder):
764 def download(self,filename,localfolder):
771 """
765 """
772 download is used to downloading file from remote folder into local folder
766 download is used to downloading file from remote folder into local folder
773
767
774 Inputs:
768 Inputs:
775 filename - filename to donwload
769 filename - filename to donwload
776
770
777 localfolder - directory local to store filename
771 localfolder - directory local to store filename
778
772
779 Returns:
773 Returns:
780 self.status - 1 in error case else 0
774 self.status - 1 in error case else 0
781 """
775 """
782
776
783 self.status = 0
777 self.status = 0
784
778
785
779
786 if not(filename in self.fileList):
780 if not(filename in self.fileList):
787 print 'filename:%s not exists'%filename
781 print 'filename:%s not exists'%filename
788 self.status = 1
782 self.status = 1
789 return self.status
783 return self.status
790
784
791 newfilename = os.path.join(localfolder,filename)
785 newfilename = os.path.join(localfolder,filename)
792
786
793 self.file = open(newfilename, 'wb')
787 self.file = open(newfilename, 'wb')
794
788
795 try:
789 try:
796 print 'Download: ' + filename
790 print 'Download: ' + filename
797 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
791 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
798 print 'Download Complete'
792 print 'Download Complete'
799 except ftplib.all_errors:
793 except ftplib.all_errors:
800 print 'Error Downloading ' + filename
794 print 'Error Downloading ' + filename
801 self.status = 1
795 self.status = 1
802 return self.status
796 return self.status
803
797
804 self.file.close()
798 self.file.close()
805
799
806 return self.status
800 return self.status
807
801
808
802
809 def __handleDownload(self,block):
803 def __handleDownload(self,block):
810 """
804 """
811 __handleDownload is used to handle writing file
805 __handleDownload is used to handle writing file
812 """
806 """
813 self.file.write(block)
807 self.file.write(block)
814
808
815
809
816 def upload(self,filename,remotefolder=None):
810 def upload(self,filename,remotefolder=None):
817 """
811 """
818 upload is used to uploading local file to remote directory
812 upload is used to uploading local file to remote directory
819
813
820 Inputs:
814 Inputs:
821 filename - full path name of local file to store in remote directory
815 filename - full path name of local file to store in remote directory
822
816
823 remotefolder - remote directory
817 remotefolder - remote directory
824
818
825 Returns:
819 Returns:
826 self.status - 1 in error case else 0
820 self.status - 1 in error case else 0
827 """
821 """
828
822
829 if remotefolder == None:
823 if remotefolder == None:
830 remotefolder = self.remotefolder
824 remotefolder = self.remotefolder
831
825
832 self.status = 0
826 self.status = 0
833
827
834 try:
828 try:
835 self.ftp.cwd(remotefolder)
829 self.ftp.cwd(remotefolder)
836
830
837 self.file = open(filename, 'rb')
831 self.file = open(filename, 'rb')
838
832
839 (head, tail) = os.path.split(filename)
833 (head, tail) = os.path.split(filename)
840
834
841 command = "STOR " + tail
835 command = "STOR " + tail
842
836
843 print 'Uploading: ' + tail
837 print 'Uploading: ' + tail
844 self.ftp.storbinary(command, self.file)
838 self.ftp.storbinary(command, self.file)
845 print 'Upload Completed'
839 print 'Upload Completed'
846
840
847 except ftplib.all_errors:
841 except ftplib.all_errors:
848 print 'Error Uploading ' + tail
842 print 'Error Uploading ' + tail
849 self.status = 1
843 self.status = 1
850 return self.status
844 return self.status
851
845
852 self.file.close()
846 self.file.close()
853
847
854 #back to initial directory in __init__()
848 #back to initial directory in __init__()
855 self.ftp.cwd(self.remotefolder)
849 self.ftp.cwd(self.remotefolder)
856
850
857 return self.status
851 return self.status
858
852
859
853
860 def dir(self,remotefolder):
854 def dir(self,remotefolder):
861 """
855 """
862 dir is used to change working directory of remote server and get folder and file list
856 dir is used to change working directory of remote server and get folder and file list
863
857
864 Input:
858 Input:
865 remotefolder - current working directory
859 remotefolder - current working directory
866
860
867 Affects:
861 Affects:
868 self.fileList - file list of working directory
862 self.fileList - file list of working directory
869
863
870 Return:
864 Return:
871 infoList - list with filenames and size of file in bytes
865 infoList - list with filenames and size of file in bytes
872
866
873 self.folderList - folder list
867 self.folderList - folder list
874 """
868 """
875
869
876 self.remotefolder = remotefolder
870 self.remotefolder = remotefolder
877 print 'Change to ' + self.remotefolder
871 print 'Change to ' + self.remotefolder
878 try:
872 try:
879 self.ftp.cwd(remotefolder)
873 self.ftp.cwd(remotefolder)
880 except ftplib.all_errors:
874 except ftplib.all_errors:
881 print 'Error Change to ' + self.remotefolder
875 print 'Error Change to ' + self.remotefolder
882 infoList = None
876 infoList = None
883 self.folderList = None
877 self.folderList = None
884 return infoList,self.folderList
878 return infoList,self.folderList
885
879
886 self.dirList = []
880 self.dirList = []
887
881
888 try:
882 try:
889 self.dirList = self.ftp.nlst()
883 self.dirList = self.ftp.nlst()
890
884
891 except ftplib.error_perm, resp:
885 except ftplib.error_perm, resp:
892 if str(resp) == "550 No files found":
886 if str(resp) == "550 No files found":
893 print "no files in this directory"
887 print "no files in this directory"
894 infoList = None
888 infoList = None
895 self.folderList = None
889 self.folderList = None
896 return infoList,self.folderList
890 return infoList,self.folderList
897 except ftplib.all_errors:
891 except ftplib.all_errors:
898 print 'Error Displaying Dir-Files'
892 print 'Error Displaying Dir-Files'
899 infoList = None
893 infoList = None
900 self.folderList = None
894 self.folderList = None
901 return infoList,self.folderList
895 return infoList,self.folderList
902
896
903 infoList = []
897 infoList = []
904 self.fileList = []
898 self.fileList = []
905 self.folderList = []
899 self.folderList = []
906 for f in self.dirList:
900 for f in self.dirList:
907 name,ext = os.path.splitext(f)
901 name,ext = os.path.splitext(f)
908 if ext != '':
902 if ext != '':
909 self.fileList.append(f)
903 self.fileList.append(f)
910 value = (f,self.ftp.size(f))
904 value = (f,self.ftp.size(f))
911 infoList.append(value)
905 infoList.append(value)
912
906
913 if ext == '':
907 if ext == '':
914 self.folderList.append(f)
908 self.folderList.append(f)
915
909
916 return infoList,self.folderList
910 return infoList,self.folderList
917
911
918
912
919 def close(self):
913 def close(self):
920 """
914 """
921 close is used to close and end FTP connection
915 close is used to close and end FTP connection
922
916
923 Inputs: None
917 Inputs: None
924
918
925 Return: void
919 Return: void
926
920
927 """
921 """
928 self.ftp.close()
922 self.ftp.close()
929
923
930 class SendByFTP(Operation):
924 class SendByFTP(Operation):
931
925
932 def __init__(self):
926 def __init__(self):
933
927
934 self.status = 1
928 self.status = 1
935 self.counter = 0
929 self.counter = 0
936
930
937 def error_print(self, ValueError):
931 def error_print(self, ValueError):
938
932
939 print ValueError, 'Error FTP'
933 print ValueError, 'Error FTP'
940 print "don't worry the program is running..."
934 print "don't worry the program is running..."
941
935
942 def worker_ftp(self, server, username, password, remotefolder, filenameList):
936 def worker_ftp(self, server, username, password, remotefolder, filenameList):
943
937
944 self.ftpClientObj = FTP(server, username, password, remotefolder)
938 self.ftpClientObj = FTP(server, username, password, remotefolder)
945 for filename in filenameList:
939 for filename in filenameList:
946 self.ftpClientObj.upload(filename)
940 self.ftpClientObj.upload(filename)
947 self.ftpClientObj.close()
941 self.ftpClientObj.close()
948
942
949 def ftp_thread(self, server, username, password, remotefolder):
943 def ftp_thread(self, server, username, password, remotefolder):
950 if not(self.status):
944 if not(self.status):
951 return
945 return
952
946
953 import multiprocessing
947 import multiprocessing
954
948
955 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
949 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
956 p.start()
950 p.start()
957
951
958 p.join(3)
952 p.join(3)
959
953
960 if p.is_alive():
954 if p.is_alive():
961 p.terminate()
955 p.terminate()
962 p.join()
956 p.join()
963 print 'killing ftp process...'
957 print 'killing ftp process...'
964 self.status = 0
958 self.status = 0
965 return
959 return
966
960
967 self.status = 1
961 self.status = 1
968 return
962 return
969
963
970 def filterByExt(self, ext, localfolder):
964 def filterByExt(self, ext, localfolder):
971 fnameList = glob.glob1(localfolder,ext)
965 fnameList = glob.glob1(localfolder,ext)
972 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
966 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
973
967
974 if len(self.filenameList) == 0:
968 if len(self.filenameList) == 0:
975 self.status = 0
969 self.status = 0
976
970
977 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
971 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
978
972
979 self.counter += 1
973 self.counter += 1
980 if self.counter >= period:
974 if self.counter >= period:
981 self.filterByExt(ext, localfolder)
975 self.filterByExt(ext, localfolder)
982
976
983 self.ftp_thread(server, username, password, remotefolder)
977 self.ftp_thread(server, username, password, remotefolder)
984
978
985 self.counter = 0
979 self.counter = 0
986
980
987 self.status = 1
981 self.status = 1
988
982
General Comments 0
You need to be logged in to leave comments. Login now