##// END OF EJS Templates
schain
RSS
minor changes and comments added
Miguel Valdez -
r641:ef17427bf2f3
parent child
Show More

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,988 +1,988
1 '''
1 '''
2 @author: Daniel Suarez
2 @author: Daniel Suarez
3 '''
3 '''
4 import os
4 import os
5 import glob
5 import glob
6 import ftplib
6 import ftplib
7
7
8 try:
8 try:
9 import paramiko
9 import paramiko
10 import scp
10 import scp
11 except:
11 except:
12 print "You should install paramiko if you will use SSH protocol to upload files to a server"
12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
13
13
14 import time
14 import time
15
15
16 import threading
16 import threading
17 Thread = threading.Thread
17 Thread = threading.Thread
18
18
19 # try:
19 # try:
20 # from gevent import sleep
20 # from gevent import sleep
21 # except:
21 # except:
22 from time import sleep
22 from time import sleep
23
23
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25
25
26 class Remote(Thread):
26 class Remote(Thread):
27 """
27 """
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 used to upload or download files remotely.
29 used to upload or download files remotely.
30
30
31 Non-standard Python modules used:
31 Non-standard Python modules used:
32 None
32 None
33
33
34 Written by:
34 Written by:
35
35
36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
37
37
38 """
38 """
39
39
40 server = None
40 server = None
41 username = None
41 username = None
42 password = None
42 password = None
43 remotefolder = None
43 remotefolder = None
44
44
45 period = 60
45 period = 60
46 fileList = []
46 fileList = []
47 bussy = False
47 bussy = False
48
48
49 def __init__(self, server, username, password, remotefolder, period=60):
49 def __init__(self, server, username, password, remotefolder, period=60):
50
50
51 Thread.__init__(self)
51 Thread.__init__(self)
52
52
53 self.setDaemon(True)
53 self.setDaemon(True)
54
54
55 self.status = 0
55 self.status = 0
56
56
57 self.period = period
57 self.period = period
58 self.fileList = []
58 self.fileList = []
59 self.bussy = False
59 self.bussy = False
60
60
61 self.stopFlag = False
61 self.stopFlag = False
62
62
63 print "[Remote Server] Opening server: %s" %server
63 print "[Remote Server] Opening server: %s" %server
64 if self.open(server, username, password, remotefolder):
64 if self.open(server, username, password, remotefolder):
65 print "[Remote Server] %s server was opened successfully" %server
65 print "[Remote Server] %s server was opened successfully" %server
66
66
67 self.mutex = threading.Lock()
67 self.mutex = threading.Lock()
68
68
69 def stop(self):
69 def stop(self):
70
70
71 self.stopFlag = True
71 self.stopFlag = True
72 self.join(10)
72 self.join(10)
73
73
74 def open(self):
74 def open(self):
75 """
75 """
76 Connect to server and create a connection class (FTP or SSH) to remote server.
76 Connect to server and create a connection class (FTP or SSH) to remote server.
77 """
77 """
78 raise NotImplementedError, "Implement this method in child class"
78 raise NotImplementedError, "Implement this method in child class"
79
79
80 def close(self):
80 def close(self):
81 """
81 """
82 Close connection to server
82 Close connection to server
83 """
83 """
84 raise NotImplementedError, "Implement this method in child class"
84 raise NotImplementedError, "Implement this method in child class"
85
85
86 def mkdir(self, remotefolder):
86 def mkdir(self, remotefolder):
87 """
87 """
88 Create a folder remotely
88 Create a folder remotely
89 """
89 """
90 raise NotImplementedError, "Implement this method in child class"
90 raise NotImplementedError, "Implement this method in child class"
91
91
92 def cd(self, remotefolder):
92 def cd(self, remotefolder):
93 """
93 """
94 Change working directory in remote server
94 Change working directory in remote server
95 """
95 """
96 raise NotImplementedError, "Implement this method in child class"
96 raise NotImplementedError, "Implement this method in child class"
97
97
98 def download(self, filename, localfolder=None):
98 def download(self, filename, localfolder=None):
99 """
99 """
100 Download a file from server to local host
100 Download a file from server to local host
101 """
101 """
102 raise NotImplementedError, "Implement this method in child class"
102 raise NotImplementedError, "Implement this method in child class"
103
103
104 def sendFile(self, fullfilename):
104 def sendFile(self, fullfilename):
105 """
105 """
106 sendFile method is used to upload a local file to the current directory in remote server
106 sendFile method is used to upload a local file to the current directory in remote server
107
107
108 Inputs:
108 Inputs:
109 fullfilename - full path name of local file to store in remote directory
109 fullfilename - full path name of local file to store in remote directory
110
110
111 Returns:
111 Returns:
112 0 in error case else 1
112 0 in error case else 1
113 """
113 """
114 raise NotImplementedError, "Implement this method in child class"
114 raise NotImplementedError, "Implement this method in child class"
115
115
116 def upload(self, fullfilename, remotefolder=None):
116 def upload(self, fullfilename, remotefolder=None):
117 """
117 """
118 upload method is used to upload a local file to remote directory. This method changes
118 upload method is used to upload a local file to remote directory. This method changes
119 working directory before sending a file.
119 working directory before sending a file.
120
120
121 Inputs:
121 Inputs:
122 fullfilename - full path name of local file to store in remote directory
122 fullfilename - full path name of local file to store in remote directory
123
123
124 remotefolder - remote directory
124 remotefolder - remote directory
125
125
126 Returns:
126 Returns:
127 0 in error case else 1
127 0 in error case else 1
128 """
128 """
129 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
129 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
130
130
131 if not self.status:
131 if not self.status:
132 return 0
132 return 0
133
133
134 if remotefolder == None:
134 if remotefolder == None:
135 remotefolder = self.remotefolder
135 remotefolder = self.remotefolder
136
136
137 if not self.cd(remotefolder):
137 if not self.cd(remotefolder):
138 return 0
138 return 0
139
139
140 if not self.sendFile(fullfilename):
140 if not self.sendFile(fullfilename):
141 print "[Remote Server] Error uploading file %s" %fullfilename
141 print "[Remote Server] Error uploading file %s" %fullfilename
142 return 0
142 return 0
143
143
144 print "[Remote Server] upload finished successfully"
144 print "[Remote Server] upload finished successfully"
145
145
146 return 1
146 return 1
147
147
148 def delete(self, filename):
148 def delete(self, filename):
149 """
149 """
150 Remove a file from remote server
150 Remove a file from remote server
151 """
151 """
152 pass
152 pass
153
153
154 def updateFileList(self, fileList):
154 def updateFileList(self, fileList):
155 """
155 """
156 Remove a file from remote server
156 Remove a file from remote server
157 """
157 """
158
158
159 if fileList == self.fileList:
159 if fileList == self.fileList:
160 return 0
160 return 0
161
161
162 self.mutex.acquire()
162 self.mutex.acquire()
163 # init = time.time()
163 # init = time.time()
164 #
164 #
165 # while(self.bussy):
165 # while(self.bussy):
166 # sleep(0.1)
166 # sleep(0.1)
167 # if time.time() - init > 2*self.period:
167 # if time.time() - init > 2*self.period:
168 # return 0
168 # return 0
169
169
170 self.fileList = fileList
170 self.fileList = fileList
171 self.mutex.release()
171 self.mutex.release()
172 return 1
172 return 1
173
173
174 def run(self):
174 def run(self):
175
175
176 if not self.status:
176 if not self.status:
177 print "Finishing FTP service"
177 print "Finishing FTP service"
178 return
178 return
179
179
180 if not self.cd(self.remotefolder):
180 if not self.cd(self.remotefolder):
181 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
181 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
182
182
183 sts = True
183 sts = True
184
184
185 while True:
185 while True:
186
186
187 for i in range(self.period):
187 for i in range(self.period):
188 if self.stopFlag:
188 if self.stopFlag:
189 break
189 break
190 sleep(1)
190 sleep(1)
191
191
192 if self.stopFlag:
192 if self.stopFlag:
193 break
193 break
194
194
195 # self.bussy = True
195 # self.bussy = True
196 self.mutex.acquire()
196 self.mutex.acquire()
197
197
198 for thisFile in self.fileList:
198 for thisFile in self.fileList:
199 sts = self.upload(thisFile, self.remotefolder)
199 sts = self.upload(thisFile, self.remotefolder)
200 if not sts: break
200 if not sts: break
201
201
202 self.mutex.release()
202 self.mutex.release()
203 # self.bussy = False
203 # self.bussy = False
204
204
205 if not sts:
205 if not sts:
206 break
206 break
207
207
208 print "[Remote Server] Thread stopped successfully"
208 print "[Remote Server] Thread stopped successfully"
209
209
210 class FTPClient(Remote):
210 class FTPClient(Remote):
211
211
212 __ftpClientObj = None
212 __ftpClientObj = None
213
213
214 def __init__(self, server, username, password, remotefolder, period=60):
214 def __init__(self, server, username, password, remotefolder, period=60):
215 """
215 """
216 """
216 """
217 Remote.__init__(self, server, username, password, remotefolder, period)
217 Remote.__init__(self, server, username, password, remotefolder, period)
218
218
219 def open(self, server, username, password, remotefolder):
219 def open(self, server, username, password, remotefolder):
220
220
221 """
221 """
222 This method is used to set FTP parameters and establish a connection to remote server
222 This method is used to set FTP parameters and establish a connection to remote server
223
223
224 Inputs:
224 Inputs:
225 server - remote server IP Address
225 server - remote server IP Address
226
226
227 username - remote server Username
227 username - remote server Username
228
228
229 password - remote server password
229 password - remote server password
230
230
231 remotefolder - remote server current working directory
231 remotefolder - remote server current working directory
232
232
233 Return: void
233 Return: void
234
234
235 Affects:
235 Affects:
236 self.status - in case of error or fail connection this parameter is set to 0 else 1
236 self.status - in case of error or fail connection this parameter is set to 0 else 1
237
237
238 """
238 """
239
239
240 if server == None:
240 if server == None:
241 raise ValueError, "FTP server should be defined"
241 raise ValueError, "FTP server should be defined"
242
242
243 if username == None:
243 if username == None:
244 raise ValueError, "FTP username should be defined"
244 raise ValueError, "FTP username should be defined"
245
245
246 if password == None:
246 if password == None:
247 raise ValueError, "FTP password should be defined"
247 raise ValueError, "FTP password should be defined"
248
248
249 if remotefolder == None:
249 if remotefolder == None:
250 raise ValueError, "FTP remote folder should be defined"
250 raise ValueError, "FTP remote folder should be defined"
251
251
252 try:
252 try:
253 ftpClientObj = ftplib.FTP(server)
253 ftpClientObj = ftplib.FTP(server)
254 except ftplib.all_errors:
254 except ftplib.all_errors:
255 print "FTP server connection fail: %s" %server
255 print "FTP server connection fail: %s" %server
256 self.status = 0
256 self.status = 0
257 return 0
257 return 0
258
258
259 try:
259 try:
260 ftpClientObj.login(username, password)
260 ftpClientObj.login(username, password)
261 except ftplib.all_errors:
261 except ftplib.all_errors:
262 print "FTP username or password are incorrect"
262 print "FTP username or password are incorrect"
263 self.status = 0
263 self.status = 0
264 return 0
264 return 0
265
265
266 if remotefolder == None:
266 if remotefolder == None:
267 remotefolder = ftpClientObj.pwd()
267 remotefolder = ftpClientObj.pwd()
268 else:
268 else:
269 try:
269 try:
270 ftpClientObj.cwd(remotefolder)
270 ftpClientObj.cwd(remotefolder)
271 except ftplib.all_errors:
271 except ftplib.all_errors:
272 print "FTP remote folder is invalid: %s" %remotefolder
272 print "FTP remote folder is invalid: %s" %remotefolder
273 remotefolder = ftpClientObj.pwd()
273 remotefolder = ftpClientObj.pwd()
274
274
275 self.server = server
275 self.server = server
276 self.username = username
276 self.username = username
277 self.password = password
277 self.password = password
278 self.remotefolder = remotefolder
278 self.remotefolder = remotefolder
279 self.__ftpClientObj = ftpClientObj
279 self.__ftpClientObj = ftpClientObj
280 self.status = 1
280 self.status = 1
281
281
282 return 1
282 return 1
283
283
284 def close(self):
284 def close(self):
285 """
285 """
286 Close connection to remote server
286 Close connection to remote server
287 """
287 """
288 if not self.status:
288 if not self.status:
289 return 0
289 return 0
290
290
291 self.__ftpClientObj.close()
291 self.__ftpClientObj.close()
292
292
293 def mkdir(self, remotefolder):
293 def mkdir(self, remotefolder):
294 """
294 """
295 mkdir is used to make a new directory in remote server
295 mkdir is used to make a new directory in remote server
296
296
297 Input:
297 Input:
298 remotefolder - directory name
298 remotefolder - directory name
299
299
300 Return:
300 Return:
301 0 in error case else 1
301 0 in error case else 1
302 """
302 """
303 if not self.status:
303 if not self.status:
304 return 0
304 return 0
305
305
306 try:
306 try:
307 self.__ftpClientObj.mkd(dirname)
307 self.__ftpClientObj.mkd(dirname)
308 except ftplib.all_errors:
308 except ftplib.all_errors:
309 print "Error creating remote folder: %s" %remotefolder
309 print "Error creating remote folder: %s" %remotefolder
310 return 0
310 return 0
311
311
312 return 1
312 return 1
313
313
314 def cd(self, remotefolder):
314 def cd(self, remotefolder):
315 """
315 """
316 cd is used to change remote working directory on server
316 cd is used to change remote working directory on server
317
317
318 Input:
318 Input:
319 remotefolder - current working directory
319 remotefolder - current working directory
320
320
321 Affects:
321 Affects:
322 self.remotefolder
322 self.remotefolder
323
323
324 Return:
324 Return:
325 0 in case of error else 1
325 0 in case of error else 1
326 """
326 """
327 if not self.status:
327 if not self.status:
328 return 0
328 return 0
329
329
330 if remotefolder == self.remotefolder:
330 if remotefolder == self.remotefolder:
331 return 1
331 return 1
332
332
333 try:
333 try:
334 self.__ftpClientObj.cwd(remotefolder)
334 self.__ftpClientObj.cwd(remotefolder)
335 except ftplib.all_errors:
335 except ftplib.all_errors:
336 print 'Error changing to %s' %remotefolder
336 print 'Error changing to %s' %remotefolder
337 print 'Trying to create remote folder'
337 print 'Trying to create remote folder'
338
338
339 if not self.mkdir(remotefolder):
339 if not self.mkdir(remotefolder):
340 print 'Remote folder could not be created'
340 print 'Remote folder could not be created'
341 return 0
341 return 0
342
342
343 try:
343 try:
344 self.__ftpClientObj.cwd(remotefolder)
344 self.__ftpClientObj.cwd(remotefolder)
345 except ftplib.all_errors:
345 except ftplib.all_errors:
346 return 0
346 return 0
347
347
348 self.remotefolder = remotefolder
348 self.remotefolder = remotefolder
349
349
350 return 1
350 return 1
351
351
352 def sendFile(self, fullfilename):
352 def sendFile(self, fullfilename):
353
353
354 if not self.status:
354 if not self.status:
355 return 0
355 return 0
356
356
357 file = open(fullfilename, 'rb')
357 file = open(fullfilename, 'rb')
358
358
359 filename = os.path.basename(fullfilename)
359 filename = os.path.basename(fullfilename)
360
360
361 command = "STOR %s" %filename
361 command = "STOR %s" %filename
362
362
363 try:
363 try:
364 self.__ftpClientObj.storbinary(command, file)
364 self.__ftpClientObj.storbinary(command, file)
365 except ftplib.all_errors:
365 except ftplib.all_errors:
366 return 0
366 return 0
367
367
368 try:
368 try:
369 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
369 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
370 except ftplib.all_errors, e:
370 except ftplib.all_errors, e:
371 print e
371 print e
372
372
373 file.close()
373 file.close()
374
374
375 return 1
375 return 1
376
376
377 class SSHClient(Remote):
377 class SSHClient(Remote):
378
378
379 __sshClientObj = None
379 __sshClientObj = None
380 __scpClientObj = None
380 __scpClientObj = None
381
381
382 def __init__(self, server, username, password, remotefolder, period=60):
382 def __init__(self, server, username, password, remotefolder, period=60):
383 """
383 """
384 """
384 """
385 Remote.__init__(self, server, username, password, remotefolder, period)
385 Remote.__init__(self, server, username, password, remotefolder, period)
386
386
387 def open(self, server, username, password, remotefolder, port=22):
387 def open(self, server, username, password, remotefolder, port=22):
388
388
389 """
389 """
390 This method is used to set SSH parameters and establish a connection to a remote server
390 This method is used to set SSH parameters and establish a connection to a remote server
391
391
392 Inputs:
392 Inputs:
393 server - remote server IP Address
393 server - remote server IP Address
394
394
395 username - remote server Username
395 username - remote server Username
396
396
397 password - remote server password
397 password - remote server password
398
398
399 remotefolder - remote server current working directory
399 remotefolder - remote server current working directory
400
400
401 Return: void
401 Return: void
402
402
403 Affects:
403 Affects:
404 self.status - in case of error or fail connection this parameter is set to 0 else 1
404 self.status - in case of error or fail connection this parameter is set to 0 else 1
405
405
406 """
406 """
407
407
408 if server == None:
408 if server == None:
409 raise ValueError, "SSH server should be defined"
409 raise ValueError, "SSH server should be defined"
410
410
411 if username == None:
411 if username == None:
412 raise ValueError, "SSH username should be defined"
412 raise ValueError, "SSH username should be defined"
413
413
414 if password == None:
414 if password == None:
415 raise ValueError, "SSH password should be defined"
415 raise ValueError, "SSH password should be defined"
416
416
417 if remotefolder == None:
417 if remotefolder == None:
418 raise ValueError, "SSH remote folder should be defined"
418 raise ValueError, "SSH remote folder should be defined"
419
419
420 try:
420 try:
421 sshClientObj = paramiko.SSHClient()
421 sshClientObj = paramiko.SSHClient()
422 except:
422 except:
423 print "SSH server connection fail: %s" %server
423 print "SSH server connection fail: %s" %server
424 self.status = 0
424 self.status = 0
425 return 0
425 return 0
426
426
427 sshClientObj.load_system_host_keys()
427 sshClientObj.load_system_host_keys()
428 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
428 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
429
429
430 try:
430 try:
431 sshClientObj.connect(server, username=username, password=password, port=port)
431 sshClientObj.connect(server, username=username, password=password, port=port)
432 except :
432 except :
433 print "SSH username or password are incorrect: %s"
433 print "SSH username or password are incorrect: %s"
434 self.status = 0
434 self.status = 0
435 return 0
435 return 0
436
436
437 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
437 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
438
438
439 if remotefolder == None:
439 if remotefolder == None:
440 remotefolder = self.pwd()
440 remotefolder = self.pwd()
441
441
442 self.server = server
442 self.server = server
443 self.username = username
443 self.username = username
444 self.password = password
444 self.password = password
445 self.__sshClientObj = sshClientObj
445 self.__sshClientObj = sshClientObj
446 self.__scpClientObj = scpClientObj
446 self.__scpClientObj = scpClientObj
447 self.status = 1
447 self.status = 1
448
448
449 if not self.cd(remotefolder):
449 if not self.cd(remotefolder):
450 raise ValueError, "Could not access to remote folder: %s" %remotefolder
450 raise ValueError, "Could not access to remote folder: %s" %remotefolder
451 return 0
451 return 0
452
452
453 self.remotefolder = remotefolder
453 self.remotefolder = remotefolder
454
454
455 return 1
455 return 1
456
456
457 def close(self):
457 def close(self):
458 """
458 """
459 Close connection to remote server
459 Close connection to remote server
460 """
460 """
461 if not self.status:
461 if not self.status:
462 return 0
462 return 0
463
463
464 self.__sshObj.close()
464 self.__sshObj.close()
465
465
466 def __execute(self, command):
466 def __execute(self, command):
467 """
467 """
468 __execute a command on remote server
468 __execute a command on remote server
469
469
470 Input:
470 Input:
471 command - Exmaple 'ls -l'
471 command - Exmaple 'ls -l'
472
472
473 Return:
473 Return:
474 0 in error case else 1
474 0 in error case else 1
475 """
475 """
476 if not self.status:
476 if not self.status:
477 return 0
477 return 0
478
478
479 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
479 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
480
480
481 result = stderr.readlines()
481 result = stderr.readlines()
482 if len(result) > 1:
482 if len(result) > 1:
483 return 0
483 return 0
484
484
485 result = stdout.readlines()
485 result = stdout.readlines()
486 if len(result) > 1:
486 if len(result) > 1:
487 return result[0][:-1]
487 return result[0][:-1]
488
488
489 return 1
489 return 1
490
490
491 def mkdir(self, remotefolder):
491 def mkdir(self, remotefolder):
492 """
492 """
493 mkdir is used to make a new directory in remote server
493 mkdir is used to make a new directory in remote server
494
494
495 Input:
495 Input:
496 remotefolder - directory name
496 remotefolder - directory name
497
497
498 Return:
498 Return:
499 0 in error case else 1
499 0 in error case else 1
500 """
500 """
501
501
502 command = 'mkdir %s' %remotefolder
502 command = 'mkdir %s' %remotefolder
503
503
504 return self.__execute(command)
504 return self.__execute(command)
505
505
506 def pwd(self):
506 def pwd(self):
507
507
508 command = 'pwd'
508 command = 'pwd'
509
509
510 return self.__execute(command)
510 return self.__execute(command)
511
511
512 def cd(self, remotefolder):
512 def cd(self, remotefolder):
513 """
513 """
514 cd is used to change remote working directory on server
514 cd is used to change remote working directory on server
515
515
516 Input:
516 Input:
517 remotefolder - current working directory
517 remotefolder - current working directory
518
518
519 Affects:
519 Affects:
520 self.remotefolder
520 self.remotefolder
521
521
522 Return:
522 Return:
523 0 in case of error else 1
523 0 in case of error else 1
524 """
524 """
525 if not self.status:
525 if not self.status:
526 return 0
526 return 0
527
527
528 if remotefolder == self.remotefolder:
528 if remotefolder == self.remotefolder:
529 return 1
529 return 1
530
530
531 chk_command = "cd %s; pwd" %remotefolder
531 chk_command = "cd %s; pwd" %remotefolder
532 mkdir_command = "mkdir %s" %remotefolder
532 mkdir_command = "mkdir %s" %remotefolder
533
533
534 if not self.__execute(chk_command):
534 if not self.__execute(chk_command):
535 if not self.__execute(mkdir_command):
535 if not self.__execute(mkdir_command):
536 self.remotefolder = None
536 self.remotefolder = None
537 return 0
537 return 0
538
538
539 self.remotefolder = remotefolder
539 self.remotefolder = remotefolder
540
540
541 return 1
541 return 1
542
542
543 def sendFile(self, fullfilename):
543 def sendFile(self, fullfilename):
544
544
545 if not self.status:
545 if not self.status:
546 return 0
546 return 0
547
547
548 try:
548 try:
549 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
549 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
550 except:
550 except:
551 return 0
551 return 0
552
552
553 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
553 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
554 command = 'chmod 775 %s' %remotefile
554 command = 'chmod 775 %s' %remotefile
555
555
556 return self.__execute(command)
556 return self.__execute(command)
557
557
558 class SendToServer(ProcessingUnit):
558 class SendToServer(ProcessingUnit):
559
559
560 def __init__(self):
560 def __init__(self):
561
561
562 ProcessingUnit.__init__(self)
562 ProcessingUnit.__init__(self)
563
563
564 self.isConfig = False
564 self.isConfig = False
565 self.clientObj = None
565 self.clientObj = None
566
566
567 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
567 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
568
568
569 self.clientObj = None
569 self.clientObj = None
570 self.localfolder = localfolder
570 self.localfolder = localfolder
571 self.ext = ext
571 self.ext = ext
572 self.period = period
572 self.period = period
573
573
574 if str.lower(protocol) == 'ftp':
574 if str.lower(protocol) == 'ftp':
575 self.clientObj = FTPClient(server, username, password, remotefolder, period)
575 self.clientObj = FTPClient(server, username, password, remotefolder, period)
576
576
577 if str.lower(protocol) == 'ssh':
577 if str.lower(protocol) == 'ssh':
578 self.clientObj = SSHClient(server, username, password, remotefolder, period)
578 self.clientObj = SSHClient(server, username, password, remotefolder, period)
579
579
580 if not self.clientObj:
580 if not self.clientObj:
581 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
581 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
582
582
583 self.clientObj.start()
583 self.clientObj.start()
584
584
585 def findFiles(self):
585 def findFiles(self):
586
586
587 if not type(self.localfolder) == list:
587 if not type(self.localfolder) == list:
588 folderList = [self.localfolder]
588 folderList = [self.localfolder]
589 else:
589 else:
590 folderList = self.localfolder
590 folderList = self.localfolder
591
591
592 #Remove duplicate items
592 #Remove duplicate items
593 folderList = list(set(folderList))
593 folderList = list(set(folderList))
594
594
595 fullfilenameList = []
595 fullfilenameList = []
596
596
597 for thisFolder in folderList:
597 for thisFolder in folderList:
598
598
599 print "[Remote Server]: Searching files on %s" %thisFolder
599 print "[Remote Server]: Searching files on %s" %thisFolder
600
600
601 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
601 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
602
602
603 if len(filenameList) < 1:
603 if len(filenameList) < 1:
604 continue
604 continue
605
605
606 for thisFile in filenameList:
606 for thisFile in filenameList:
607 fullfilename = os.path.join(thisFolder, thisFile)
607 fullfilename = os.path.join(thisFolder, thisFile)
608
608
609 if fullfilename in fullfilenameList:
609 if fullfilename in fullfilenameList:
610 continue
610 continue
611
611
612 #Only files modified in the last 30 minutes are considered
612 #Only files modified in the last 30 minutes are considered
613 if os.path.getmtime(fullfilename) < time.time() - 30*60:
613 if os.path.getmtime(fullfilename) < time.time() - 30*60:
614 continue
614 continue
615
615
616 fullfilenameList.append(fullfilename)
616 fullfilenameList.append(fullfilename)
617
617
618 return fullfilenameList
618 return fullfilenameList
619
619
620 def run(self, **kwargs):
620 def run(self, **kwargs):
621
621
622 if not self.isConfig:
622 if not self.isConfig:
623 self.init = time.time()
623 self.init = time.time()
624 self.setup(**kwargs)
624 self.setup(**kwargs)
625 self.isConfig = True
625 self.isConfig = True
626
626
627 if time.time() - self.init >= self.period:
627 if time.time() - self.init >= self.period:
628 fullfilenameList = self.findFiles()
628 fullfilenameList = self.findFiles()
629
629
630 if self.clientObj.updateFileList(fullfilenameList):
630 if self.clientObj.updateFileList(fullfilenameList):
631 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
631 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
632
632
633 self.init = time.time()
633 self.init = time.time()
634
634
635 def close(self):
635 def close(self):
636 print "[Remote Server] Stopping thread"
636 print "[Remote Server] Stopping thread"
637 self.clientObj.stop()
637 self.clientObj.stop()
638
638
639
639
640 class FTP(object):
640 class FTP(object):
641 """
641 """
642 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
642 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
643
643
644 Non-standard Python modules used: None
644 Non-standard Python modules used: None
645
645
646 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
646 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
647 """
647 """
648
648
649 def __init__(self,server = None, username=None, password=None, remotefolder=None):
649 def __init__(self,server = None, username=None, password=None, remotefolder=None):
650 """
650 """
651 This method is used to setting parameters for FTP and establishing connection to remote server
651 This method is used to setting parameters for FTP and establishing connection to remote server
652
652
653 Inputs:
653 Inputs:
654 server - remote server IP Address
654 server - remote server IP Address
655
655
656 username - remote server Username
656 username - remote server Username
657
657
658 password - remote server password
658 password - remote server password
659
659
660 remotefolder - remote server current working directory
660 remotefolder - remote server current working directory
661
661
662 Return: void
662 Return: void
663
663
664 Affects:
664 Affects:
665 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
665 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
666
666
667 self.folderList - sub-folder list of remote folder
667 self.folderList - sub-folder list of remote folder
668
668
669 self.fileList - file list of remote folder
669 self.fileList - file list of remote folder
670
670
671
671
672 """
672 """
673
673
674 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
674 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
675 server, username, password, remotefolder = self.parmsByDefault()
675 server, username, password, remotefolder = self.parmsByDefault()
676
676
677 self.server = server
677 self.server = server
678 self.username = username
678 self.username = username
679 self.password = password
679 self.password = password
680 self.remotefolder = remotefolder
680 self.remotefolder = remotefolder
681 self.file = None
681 self.file = None
682 self.ftp = None
682 self.ftp = None
683 self.status = 0
683 self.status = 0
684
684
685 try:
685 try:
686 self.ftp = ftplib.FTP(self.server)
686 self.ftp = ftplib.FTP(self.server)
687 self.ftp.login(self.username,self.password)
687 self.ftp.login(self.username,self.password)
688 self.ftp.cwd(self.remotefolder)
688 self.ftp.cwd(self.remotefolder)
689 # print 'Connect to FTP Server: Successfully'
689 # print 'Connect to FTP Server: Successfully'
690
690
691 except ftplib.all_errors:
691 except ftplib.all_errors:
692 print 'Error FTP Service'
692 print 'Error FTP Service'
693 self.status = 1
693 self.status = 1
694 return
694 return
695
695
696
696
697
697
698 self.dirList = []
698 self.dirList = []
699
699
700 try:
700 try:
701 self.dirList = self.ftp.nlst()
701 self.dirList = self.ftp.nlst()
702
702
703 except ftplib.error_perm, resp:
703 except ftplib.error_perm, resp:
704 if str(resp) == "550 No files found":
704 if str(resp) == "550 No files found":
705 print "no files in this directory"
705 print "no files in this directory"
706 self.status = 1
706 self.status = 1
707 return
707 return
708
708
709 except ftplib.all_errors:
709 except ftplib.all_errors:
710 print 'Error Displaying Dir-Files'
710 print 'Error Displaying Dir-Files'
711 self.status = 1
711 self.status = 1
712 return
712 return
713
713
714 self.fileList = []
714 self.fileList = []
715 self.folderList = []
715 self.folderList = []
716 #only for test
716 #only for test
717 for f in self.dirList:
717 for f in self.dirList:
718 name, ext = os.path.splitext(f)
718 name, ext = os.path.splitext(f)
719 if ext != '':
719 if ext != '':
720 self.fileList.append(f)
720 self.fileList.append(f)
721 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
721 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
722
722
723 def parmsByDefault(self):
723 def parmsByDefault(self):
724 server = 'jro-app.igp.gob.pe'
724 server = 'jro-app.igp.gob.pe'
725 username = 'wmaster'
725 username = 'wmaster'
726 password = 'mst2010vhf'
726 password = 'mst2010vhf'
727 remotefolder = '/home/wmaster/graficos'
727 remotefolder = '/home/wmaster/graficos'
728
728
729 return server, username, password, remotefolder
729 return server, username, password, remotefolder
730
730
731
731
732 def mkd(self,dirname):
732 def mkd(self,dirname):
733 """
733 """
734 mkd is used to make directory in remote server
734 mkd is used to make directory in remote server
735
735
736 Input:
736 Input:
737 dirname - directory name
737 dirname - directory name
738
738
739 Return:
739 Return:
740 1 in error case else 0
740 1 in error case else 0
741 """
741 """
742 try:
742 try:
743 self.ftp.mkd(dirname)
743 self.ftp.mkd(dirname)
744 except:
744 except:
745 print 'Error creating remote folder:%s'%dirname
745 print 'Error creating remote folder:%s'%dirname
746 return 1
746 return 1
747
747
748 return 0
748 return 0
749
749
750
750
751 def delete(self,filename):
751 def delete(self,filename):
752 """
752 """
753 delete is used to delete file in current working directory of remote server
753 delete is used to delete file in current working directory of remote server
754
754
755 Input:
755 Input:
756 filename - filename to delete in remote folder
756 filename - filename to delete in remote folder
757
757
758 Return:
758 Return:
759 1 in error case else 0
759 1 in error case else 0
760 """
760 """
761
761
762 try:
762 try:
763 self.ftp.delete(filename)
763 self.ftp.delete(filename)
764 except:
764 except:
765 print 'Error deleting remote file:%s'%filename
765 print 'Error deleting remote file:%s'%filename
766 return 1
766 return 1
767
767
768 return 0
768 return 0
769
769
770 def download(self,filename,localfolder):
770 def download(self,filename,localfolder):
771 """
771 """
772 download is used to downloading file from remote folder into local folder
772 download is used to downloading file from remote folder into local folder
773
773
774 Inputs:
774 Inputs:
775 filename - filename to donwload
775 filename - filename to donwload
776
776
777 localfolder - directory local to store filename
777 localfolder - directory local to store filename
778
778
779 Returns:
779 Returns:
780 self.status - 1 in error case else 0
780 self.status - 1 in error case else 0
781 """
781 """
782
782
783 self.status = 0
783 self.status = 0
784
784
785
785
786 if not(filename in self.fileList):
786 if not(filename in self.fileList):
787 print 'filename:%s not exists'%filename
787 print 'filename:%s not exists'%filename
788 self.status = 1
788 self.status = 1
789 return self.status
789 return self.status
790
790
791 newfilename = os.path.join(localfolder,filename)
791 newfilename = os.path.join(localfolder,filename)
792
792
793 self.file = open(newfilename, 'wb')
793 self.file = open(newfilename, 'wb')
794
794
795 try:
795 try:
796 print 'Download: ' + filename
796 print 'Download: ' + filename
797 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
797 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
798 print 'Download Complete'
798 print 'Download Complete'
799 except ftplib.all_errors:
799 except ftplib.all_errors:
800 print 'Error Downloading ' + filename
800 print 'Error Downloading ' + filename
801 self.status = 1
801 self.status = 1
802 return self.status
802 return self.status
803
803
804 self.file.close()
804 self.file.close()
805
805
806 return self.status
806 return self.status
807
807
808
808
809 def __handleDownload(self,block):
809 def __handleDownload(self,block):
810 """
810 """
811 __handleDownload is used to handle writing file
811 __handleDownload is used to handle writing file
812 """
812 """
813 self.file.write(block)
813 self.file.write(block)
814
814
815
815
816 def upload(self,filename,remotefolder=None):
816 def upload(self,filename,remotefolder=None):
817 """
817 """
818 upload is used to uploading local file to remote directory
818 upload is used to uploading local file to remote directory
819
819
820 Inputs:
820 Inputs:
821 filename - full path name of local file to store in remote directory
821 filename - full path name of local file to store in remote directory
822
822
823 remotefolder - remote directory
823 remotefolder - remote directory
824
824
825 Returns:
825 Returns:
826 self.status - 1 in error case else 0
826 self.status - 1 in error case else 0
827 """
827 """
828
828
829 if remotefolder == None:
829 if remotefolder == None:
830 remotefolder = self.remotefolder
830 remotefolder = self.remotefolder
831
831
832 self.status = 0
832 self.status = 0
833
833
834 try:
834 try:
835 self.ftp.cwd(remotefolder)
835 self.ftp.cwd(remotefolder)
836
836
837 self.file = open(filename, 'rb')
837 self.file = open(filename, 'rb')
838
838
839 (head, tail) = os.path.split(filename)
839 (head, tail) = os.path.split(filename)
840
840
841 command = "STOR " + tail
841 command = "STOR " + tail
842
842
843 print 'Uploading: ' + tail
843 print 'Uploading: ' + tail
844 self.ftp.storbinary(command, self.file)
844 self.ftp.storbinary(command, self.file)
845 print 'Upload Completed'
845 print 'Upload Completed'
846
846
847 except ftplib.all_errors:
847 except ftplib.all_errors:
848 print 'Error Uploading ' + tail
848 print 'Error Uploading ' + tail
849 self.status = 1
849 self.status = 1
850 return self.status
850 return self.status
851
851
852 self.file.close()
852 self.file.close()
853
853
854 #back to initial directory in __init__()
854 #back to initial directory in __init__()
855 self.ftp.cwd(self.remotefolder)
855 self.ftp.cwd(self.remotefolder)
856
856
857 return self.status
857 return self.status
858
858
859
859
860 def dir(self,remotefolder):
860 def dir(self,remotefolder):
861 """
861 """
862 dir is used to change working directory of remote server and get folder and file list
862 dir is used to change working directory of remote server and get folder and file list
863
863
864 Input:
864 Input:
865 remotefolder - current working directory
865 remotefolder - current working directory
866
866
867 Affects:
867 Affects:
868 self.fileList - file list of working directory
868 self.fileList - file list of working directory
869
869
870 Return:
870 Return:
871 infoList - list with filenames and size of file in bytes
871 infoList - list with filenames and size of file in bytes
872
872
873 self.folderList - folder list
873 self.folderList - folder list
874 """
874 """
875
875
876 self.remotefolder = remotefolder
876 self.remotefolder = remotefolder
877 print 'Change to ' + self.remotefolder
877 print 'Change to ' + self.remotefolder
878 try:
878 try:
879 self.ftp.cwd(remotefolder)
879 self.ftp.cwd(remotefolder)
880 except ftplib.all_errors:
880 except ftplib.all_errors:
881 print 'Error Change to ' + self.remotefolder
881 print 'Error Change to ' + self.remotefolder
882 infoList = None
882 infoList = None
883 self.folderList = None
883 self.folderList = None
884 return infoList,self.folderList
884 return infoList,self.folderList
885
885
886 self.dirList = []
886 self.dirList = []
887
887
888 try:
888 try:
889 self.dirList = self.ftp.nlst()
889 self.dirList = self.ftp.nlst()
890
890
891 except ftplib.error_perm, resp:
891 except ftplib.error_perm, resp:
892 if str(resp) == "550 No files found":
892 if str(resp) == "550 No files found":
893 print "no files in this directory"
893 print "no files in this directory"
894 infoList = None
894 infoList = None
895 self.folderList = None
895 self.folderList = None
896 return infoList,self.folderList
896 return infoList,self.folderList
897 except ftplib.all_errors:
897 except ftplib.all_errors:
898 print 'Error Displaying Dir-Files'
898 print 'Error Displaying Dir-Files'
899 infoList = None
899 infoList = None
900 self.folderList = None
900 self.folderList = None
901 return infoList,self.folderList
901 return infoList,self.folderList
902
902
903 infoList = []
903 infoList = []
904 self.fileList = []
904 self.fileList = []
905 self.folderList = []
905 self.folderList = []
906 for f in self.dirList:
906 for f in self.dirList:
907 name,ext = os.path.splitext(f)
907 name,ext = os.path.splitext(f)
908 if ext != '':
908 if ext != '':
909 self.fileList.append(f)
909 self.fileList.append(f)
910 value = (f,self.ftp.size(f))
910 value = (f,self.ftp.size(f))
911 infoList.append(value)
911 infoList.append(value)
912
912
913 if ext == '':
913 if ext == '':
914 self.folderList.append(f)
914 self.folderList.append(f)
915
915
916 return infoList,self.folderList
916 return infoList,self.folderList
917
917
918
918
919 def close(self):
919 def close(self):
920 """
920 """
921 close is used to close and end FTP connection
921 close is used to close and end FTP connection
922
922
923 Inputs: None
923 Inputs: None
924
924
925 Return: void
925 Return: void
926
926
927 """
927 """
928 self.ftp.close()
928 self.ftp.close()
929
929
930 class SendByFTP(Operation):
930 class SendByFTP(Operation):
931
931
932 def __init__(self):
932 def __init__(self):
933
933
934 self.status = 1
934 self.status = 1
935 self.counter = 0
935 self.counter = 0
936
936
937 def error_print(self, ValueError):
937 def error_print(self, ValueError):
938
938
939 print ValueError, 'Error FTP'
939 print ValueError, 'Error FTP'
940 print "don't worry the program is running..."
940 print "don't worry the program is running..."
941
941
942 def worker_ftp(self, server, username, password, remotefolder, filenameList):
942 def worker_ftp(self, server, username, password, remotefolder, filenameList):
943
943
944 self.ftpClientObj = FTP(server, username, password, remotefolder)
944 self.ftpClientObj = FTP(server, username, password, remotefolder)
945 for filename in filenameList:
945 for filename in filenameList:
946 self.ftpClientObj.upload(filename)
946 self.ftpClientObj.upload(filename)
947 self.ftpClientObj.close()
947 self.ftpClientObj.close()
948
948
949 def ftp_thread(self, server, username, password, remotefolder):
949 def ftp_thread(self, server, username, password, remotefolder):
950 if not(self.status):
950 if not(self.status):
951 return
951 return
952
952
953 import multiprocessing
953 import multiprocessing
954
954
955 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
955 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
956 p.start()
956 p.start()
957
957
958 p.join(3)
958 p.join(3)
959
959
960 if p.is_alive():
960 if p.is_alive():
961 p.terminate()
961 p.terminate()
962 p.join()
962 p.join()
963 print 'killing ftp process...'
963 print 'killing ftp process...'
964 self.status = 0
964 self.status = 0
965 return
965 return
966
966
967 self.status = 1
967 self.status = 1
968 return
968 return
969
969
970 def filterByExt(self, ext, localfolder):
970 def filterByExt(self, ext, localfolder):
971 fnameList = glob.glob1(localfolder,ext)
971 fnameList = glob.glob1(localfolder,ext)
972 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
972 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
973
973
974 if len(self.filenameList) == 0:
974 if len(self.filenameList) == 0:
975 self.status = 0
975 self.status = 0
976
976
977 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
977 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
978
978
979 self.counter += 1
979 self.counter += 1
980 if self.counter >= period:
980 if self.counter >= period:
981 self.filterByExt(ext, localfolder)
981 self.filterByExt(ext, localfolder)
982
982
983 self.ftp_thread(server, username, password, remotefolder)
983 self.ftp_thread(server, username, password, remotefolder)
984
984
985 self.counter = 0
985 self.counter = 0
986
986
987 self.status = 1
987 self.status = 1
988
988
General Comments 0
You need to be logged in to leave comments. Login now