jroutils_ftp.py
1008 lines
| 26.5 KiB
| text/x-python
|
PythonLexer
|
r488 | ''' | ||
@author: Daniel Suarez | ||||
''' | ||||
import os | ||||
import glob | ||||
import ftplib | ||||
|
r573 | |||
try: | ||||
import paramiko | ||||
import scp | ||||
except: | ||||
|
r641 | print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server" | ||
|
r573 | |||
import time | ||||
|
r633 | import threading | ||
Thread = threading.Thread | ||||
|
r897 | |||
|
r633 | # try: | ||
# from gevent import sleep | ||||
# except: | ||||
from time import sleep | ||||
|
r897 | |||
|
r568 | from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation | ||
|
r488 | |||
|
r633 | class Remote(Thread): | ||
|
r573 | """ | ||
Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are | ||||
used to upload or download files remotely. | ||||
|
r897 | |||
|
r573 | Non-standard Python modules used: | ||
None | ||||
|
r897 | |||
|
r573 | Written by: | ||
|
r1026 | "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015 | ||
|
r573 | """ | ||
|
r897 | |||
|
r573 | server = None | ||
username = None | ||||
password = None | ||||
remotefolder = None | ||||
|
r897 | |||
|
r573 | period = 60 | ||
fileList = [] | ||||
bussy = False | ||||
|
r897 | |||
|
r573 | def __init__(self, server, username, password, remotefolder, period=60): | ||
|
r897 | |||
|
r633 | Thread.__init__(self) | ||
|
r897 | |||
|
r606 | self.setDaemon(True) | ||
|
r897 | |||
|
r573 | self.status = 0 | ||
|
r653 | self.__server = server | ||
self.__username = username | ||||
self.__password = password | ||||
self.__remotefolder = remotefolder | ||||
|
r897 | |||
|
r573 | self.period = period | ||
|
r897 | |||
|
r573 | self.fileList = [] | ||
self.bussy = False | ||||
|
r897 | |||
|
r573 | self.stopFlag = False | ||
|
r897 | |||
|
r653 | print "[Remote Server] Opening server: %s" %self.__server | ||
if self.open(self.__server, self.__username, self.__password, self.__remotefolder): | ||||
print "[Remote Server] %s server was opened successfully" %self.__server | ||||
|
r897 | |||
|
r653 | self.close() | ||
|
r897 | |||
|
r633 | self.mutex = threading.Lock() | ||
|
r897 | |||
|
r573 | def stop(self): | ||
|
r897 | |||
|
r573 | self.stopFlag = True | ||
|
r633 | self.join(10) | ||
|
r897 | |||
|
r573 | def open(self): | ||
""" | ||||
Connect to server and create a connection class (FTP or SSH) to remote server. | ||||
""" | ||||
raise NotImplementedError, "Implement this method in child class" | ||||
|
r897 | |||
|
r573 | def close(self): | ||
""" | ||||
Close connection to server | ||||
""" | ||||
raise NotImplementedError, "Implement this method in child class" | ||||
|
r897 | |||
|
r573 | def mkdir(self, remotefolder): | ||
""" | ||||
Create a folder remotely | ||||
""" | ||||
raise NotImplementedError, "Implement this method in child class" | ||||
|
r897 | |||
|
r573 | def cd(self, remotefolder): | ||
""" | ||||
Change working directory in remote server | ||||
""" | ||||
raise NotImplementedError, "Implement this method in child class" | ||||
|
r897 | |||
|
r573 | def download(self, filename, localfolder=None): | ||
""" | ||||
Download a file from server to local host | ||||
""" | ||||
raise NotImplementedError, "Implement this method in child class" | ||||
|
r897 | |||
|
r573 | def sendFile(self, fullfilename): | ||
""" | ||||
sendFile method is used to upload a local file to the current directory in remote server | ||||
|
r897 | |||
|
r573 | Inputs: | ||
fullfilename - full path name of local file to store in remote directory | ||||
|
r897 | |||
|
r573 | Returns: | ||
0 in error case else 1 | ||||
""" | ||||
raise NotImplementedError, "Implement this method in child class" | ||||
|
r897 | |||
def upload(self, fullfilename, remotefolder=None): | ||||
|
r573 | """ | ||
upload method is used to upload a local file to remote directory. This method changes | ||||
working directory before sending a file. | ||||
|
r897 | |||
|
r573 | Inputs: | ||
fullfilename - full path name of local file to store in remote directory | ||||
|
r897 | |||
remotefolder - remote directory | ||||
|
r573 | Returns: | ||
0 in error case else 1 | ||||
""" | ||||
print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder) | ||||
|
r897 | |||
|
r573 | if not self.status: | ||
return 0 | ||||
|
r897 | |||
|
r573 | if remotefolder == None: | ||
remotefolder = self.remotefolder | ||||
|
r897 | |||
|
r573 | if not self.cd(remotefolder): | ||
return 0 | ||||
|
r897 | |||
|
r573 | if not self.sendFile(fullfilename): | ||
print "[Remote Server] Error uploading file %s" %fullfilename | ||||
return 0 | ||||
|
r897 | |||
|
r573 | print "[Remote Server] upload finished successfully" | ||
|
r897 | |||
|
r573 | return 1 | ||
|
r897 | |||
|
r573 | def delete(self, filename): | ||
""" | ||||
Remove a file from remote server | ||||
""" | ||||
pass | ||||
|
r897 | |||
|
r573 | def updateFileList(self, fileList): | ||
""" | ||||
Remove a file from remote server | ||||
""" | ||||
|
r897 | |||
|
r573 | if fileList == self.fileList: | ||
|
r606 | return 0 | ||
|
r897 | |||
|
r633 | self.mutex.acquire() | ||
|
r1026 | # init = time.time() | ||
# | ||||
# while(self.bussy): | ||||
# sleep(0.1) | ||||
# if time.time() - init > 2*self.period: | ||||
# return 0 | ||||
|
r573 | self.fileList = fileList | ||
|
r633 | self.mutex.release() | ||
|
r573 | return 1 | ||
|
r897 | |||
|
r573 | def run(self): | ||
|
r897 | |||
|
r580 | if not self.status: | ||
print "Finishing FTP service" | ||||
return | ||||
|
r897 | |||
|
r573 | if not self.cd(self.remotefolder): | ||
|
r576 | raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder | ||
|
r897 | |||
|
r573 | while True: | ||
|
r897 | |||
|
r633 | for i in range(self.period): | ||
if self.stopFlag: | ||||
break | ||||
sleep(1) | ||||
|
r897 | |||
|
r633 | if self.stopFlag: | ||
break | ||||
|
r1034 | # self.bussy = True | ||
|
r633 | self.mutex.acquire() | ||
|
r897 | |||
|
r653 | print "[Remote Server] Opening %s" %self.__server | ||
if not self.open(self.__server, self.__username, self.__password, self.__remotefolder): | ||||
|
r661 | self.mutex.release() | ||
|
r662 | continue | ||
|
r897 | |||
|
r573 | for thisFile in self.fileList: | ||
|
r646 | self.upload(thisFile, self.remotefolder) | ||
|
r897 | |||
|
r653 | print "[Remote Server] Closing %s" %self.__server | ||
self.close() | ||||
|
r897 | |||
|
r633 | self.mutex.release() | ||
|
r1034 | # self.bussy = False | ||
|
r897 | |||
|
r573 | print "[Remote Server] Thread stopped successfully" | ||
|
r897 | |||
|
r573 | class FTPClient(Remote): | ||
|
r897 | |||
|
r573 | __ftpClientObj = None | ||
|
r897 | |||
|
r573 | def __init__(self, server, username, password, remotefolder, period=60): | ||
""" | ||||
""" | ||||
Remote.__init__(self, server, username, password, remotefolder, period) | ||||
|
r897 | |||
|
r573 | def open(self, server, username, password, remotefolder): | ||
|
r897 | |||
|
r573 | """ | ||
This method is used to set FTP parameters and establish a connection to remote server | ||||
|
r897 | |||
|
r573 | Inputs: | ||
|
r897 | server - remote server IP Address | ||
username - remote server Username | ||||
|
r573 | password - remote server password | ||
|
r897 | |||
|
r573 | remotefolder - remote server current working directory | ||
|
r897 | |||
|
r653 | Return: | ||
Boolean - Returns 1 if a connection has been established, 0 otherwise | ||||
|
r897 | |||
Affects: | ||||
|
r573 | self.status - in case of error or fail connection this parameter is set to 0 else 1 | ||
""" | ||||
|
r897 | |||
|
r573 | if server == None: | ||
raise ValueError, "FTP server should be defined" | ||||
|
r897 | |||
|
r573 | if username == None: | ||
raise ValueError, "FTP username should be defined" | ||||
|
r897 | |||
|
r573 | if password == None: | ||
raise ValueError, "FTP password should be defined" | ||||
|
r897 | |||
|
r573 | if remotefolder == None: | ||
raise ValueError, "FTP remote folder should be defined" | ||||
|
r897 | |||
|
r573 | try: | ||
ftpClientObj = ftplib.FTP(server) | ||||
|
r653 | except ftplib.all_errors, e: | ||
print "[FTP Server]: FTP server connection fail: %s" %server | ||||
print "[FTP Server]:", e | ||||
|
r573 | self.status = 0 | ||
return 0 | ||||
|
r897 | |||
|
r573 | try: | ||
ftpClientObj.login(username, password) | ||||
except ftplib.all_errors: | ||||
|
r653 | print "[FTP Server]: FTP username or password are incorrect" | ||
|
r573 | self.status = 0 | ||
return 0 | ||||
|
r897 | |||
|
r573 | if remotefolder == None: | ||
remotefolder = ftpClientObj.pwd() | ||||
else: | ||||
try: | ||||
ftpClientObj.cwd(remotefolder) | ||||
except ftplib.all_errors: | ||||
|
r653 | print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder | ||
|
r573 | remotefolder = ftpClientObj.pwd() | ||
|
r897 | |||
|
r573 | self.server = server | ||
self.username = username | ||||
self.password = password | ||||
self.remotefolder = remotefolder | ||||
self.__ftpClientObj = ftpClientObj | ||||
self.status = 1 | ||||
|
r897 | |||
|
r573 | return 1 | ||
|
r897 | |||
|
r573 | def close(self): | ||
""" | ||||
Close connection to remote server | ||||
""" | ||||
if not self.status: | ||||
return 0 | ||||
|
r897 | |||
|
r573 | self.__ftpClientObj.close() | ||
|
r897 | |||
|
r573 | def mkdir(self, remotefolder): | ||
""" | ||||
mkdir is used to make a new directory in remote server | ||||
|
r897 | |||
|
r573 | Input: | ||
remotefolder - directory name | ||||
|
r897 | |||
|
r573 | Return: | ||
0 in error case else 1 | ||||
""" | ||||
if not self.status: | ||||
return 0 | ||||
|
r897 | |||
|
r573 | try: | ||
self.__ftpClientObj.mkd(dirname) | ||||
except ftplib.all_errors: | ||||
|
r653 | print "[FTP Server]: Error creating remote folder: %s" %remotefolder | ||
|
r573 | return 0 | ||
|
r897 | |||
|
r573 | return 1 | ||
|
r897 | |||
|
r573 | def cd(self, remotefolder): | ||
""" | ||||
cd is used to change remote working directory on server | ||||
|
r897 | |||
|
r573 | Input: | ||
remotefolder - current working directory | ||||
|
r897 | |||
|
r573 | Affects: | ||
self.remotefolder | ||||
|
r897 | |||
Return: | ||||
|
r573 | 0 in case of error else 1 | ||
""" | ||||
if not self.status: | ||||
return 0 | ||||
|
r897 | |||
|
r573 | if remotefolder == self.remotefolder: | ||
return 1 | ||||
|
r897 | |||
try: | ||||
self.__ftpClientObj.cwd(remotefolder) | ||||
|
r573 | except ftplib.all_errors: | ||
|
r653 | print '[FTP Server]: Error changing to %s' %remotefolder | ||
print '[FTP Server]: Trying to create remote folder' | ||||
|
r897 | |||
|
r573 | if not self.mkdir(remotefolder): | ||
|
r653 | print '[FTP Server]: Remote folder could not be created' | ||
|
r573 | return 0 | ||
|
r897 | |||
try: | ||||
self.__ftpClientObj.cwd(remotefolder) | ||||
|
r573 | except ftplib.all_errors: | ||
return 0 | ||||
|
r897 | |||
|
r573 | self.remotefolder = remotefolder | ||
|
r897 | |||
|
r573 | return 1 | ||
|
r897 | |||
|
r573 | def sendFile(self, fullfilename): | ||
if not self.status: | ||||
return 0 | ||||
|
r897 | |||
|
r653 | fp = open(fullfilename, 'rb') | ||
|
r897 | |||
|
r606 | filename = os.path.basename(fullfilename) | ||
|
r897 | |||
|
r573 | command = "STOR %s" %filename | ||
|
r897 | |||
|
r573 | try: | ||
|
r653 | self.__ftpClientObj.storbinary(command, fp) | ||
except ftplib.all_errors, e: | ||||
print "[FTP Server]:", e | ||||
|
r573 | return 0 | ||
|
r897 | |||
|
r573 | try: | ||
self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename) | ||||
except ftplib.all_errors, e: | ||||
|
r653 | print "[FTP Server]:", e | ||
|
r897 | |||
|
r653 | fp.close() | ||
|
r897 | |||
|
r573 | return 1 | ||
class SSHClient(Remote): | ||||
|
r897 | |||
|
r573 | __sshClientObj = None | ||
__scpClientObj = None | ||||
|
r897 | |||
|
r573 | def __init__(self, server, username, password, remotefolder, period=60): | ||
""" | ||||
""" | ||||
Remote.__init__(self, server, username, password, remotefolder, period) | ||||
|
r897 | |||
|
r573 | def open(self, server, username, password, remotefolder, port=22): | ||
|
r897 | |||
|
r573 | """ | ||
|
r1026 | This method is used to set SSH parameters and establish a connection to a remote server | ||
|
r573 | |||
|
r1026 | Inputs: | ||
server - remote server IP Address | ||||
username - remote server Username | ||||
password - remote server password | ||||
remotefolder - remote server current working directory | ||||
|
r573 | |||
|
r1026 | Return: void | ||
|
r573 | |||
|
r1026 | Affects: | ||
self.status - in case of error or fail connection this parameter is set to 0 else 1 | ||||
|
r573 | |||
""" | ||||
|
r653 | import socket | ||
|
r897 | |||
|
r573 | if server == None: | ||
raise ValueError, "SSH server should be defined" | ||||
|
r897 | |||
|
r573 | if username == None: | ||
raise ValueError, "SSH username should be defined" | ||||
|
r897 | |||
|
r573 | if password == None: | ||
raise ValueError, "SSH password should be defined" | ||||
|
r897 | |||
|
r573 | if remotefolder == None: | ||
raise ValueError, "SSH remote folder should be defined" | ||||
|
r897 | |||
|
r653 | sshClientObj = paramiko.SSHClient() | ||
|
r897 | |||
|
r573 | sshClientObj.load_system_host_keys() | ||
sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy()) | ||||
|
r897 | |||
|
r653 | self.status = 0 | ||
|
r573 | try: | ||
|
r897 | sshClientObj.connect(server, username=username, password=password, port=port) | ||
|
r653 | except paramiko.AuthenticationException, e: | ||
|
r1026 | # print "SSH username or password are incorrect: %s" | ||
|
r653 | print "[SSH Server]:", e | ||
return 0 | ||||
except SSHException, e: | ||||
print "[SSH Server]:", e | ||||
return 0 | ||||
except socket.error: | ||||
|
r573 | self.status = 0 | ||
|
r653 | print "[SSH Server]:", e | ||
|
r573 | return 0 | ||
|
r897 | |||
|
r653 | self.status = 1 | ||
|
r573 | scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30) | ||
|
r897 | |||
|
r573 | if remotefolder == None: | ||
remotefolder = self.pwd() | ||||
|
r897 | |||
|
r573 | self.server = server | ||
self.username = username | ||||
self.password = password | ||||
self.__sshClientObj = sshClientObj | ||||
self.__scpClientObj = scpClientObj | ||||
self.status = 1 | ||||
|
r897 | |||
|
r576 | if not self.cd(remotefolder): | ||
|
r653 | raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder | ||
|
r576 | return 0 | ||
|
r897 | |||
|
r576 | self.remotefolder = remotefolder | ||
|
r897 | |||
|
r573 | return 1 | ||
|
r897 | |||
|
r573 | def close(self): | ||
""" | ||||
|
r1026 | Close connection to remote server | ||
|
r573 | """ | ||
if not self.status: | ||||
return 0 | ||||
|
r897 | |||
|
r653 | self.__scpClientObj.close() | ||
self.__sshClientObj.close() | ||||
|
r576 | |||
def __execute(self, command): | ||||
|
r573 | """ | ||
|
r1026 | __execute a command on remote server | ||
Input: | ||||
command - Exmaple 'ls -l' | ||||
Return: | ||||
0 in error case else 1 | ||||
|
r573 | """ | ||
if not self.status: | ||||
return 0 | ||||
|
r576 | |||
stdin, stdout, stderr = self.__sshClientObj.exec_command(command) | ||||
|
r897 | |||
|
r576 | result = stderr.readlines() | ||
|
r573 | if len(result) > 1: | ||
return 0 | ||||
|
r576 | |||
result = stdout.readlines() | ||||
if len(result) > 1: | ||||
return result[0][:-1] | ||||
|
r897 | |||
|
r573 | return 1 | ||
|
r897 | |||
|
r576 | def mkdir(self, remotefolder): | ||
""" | ||||
|
r1026 | mkdir is used to make a new directory in remote server | ||
Input: | ||||
remotefolder - directory name | ||||
Return: | ||||
0 in error case else 1 | ||||
|
r576 | """ | ||
|
r897 | |||
|
r576 | command = 'mkdir %s' %remotefolder | ||
|
r897 | |||
|
r576 | return self.__execute(command) | ||
|
r897 | |||
|
r573 | def pwd(self): | ||
|
r576 | command = 'pwd' | ||
|
r897 | |||
|
r576 | return self.__execute(command) | ||
|
r897 | |||
|
r573 | def cd(self, remotefolder): | ||
""" | ||||
|
r1026 | cd is used to change remote working directory on server | ||
|
r573 | |||
|
r1026 | Input: | ||
remotefolder - current working directory | ||||
Affects: | ||||
self.remotefolder | ||||
Return: | ||||
0 in case of error else 1 | ||||
|
r573 | """ | ||
if not self.status: | ||||
return 0 | ||||
|
r576 | |||
|
r573 | if remotefolder == self.remotefolder: | ||
return 1 | ||||
|
r897 | |||
|
r576 | chk_command = "cd %s; pwd" %remotefolder | ||
mkdir_command = "mkdir %s" %remotefolder | ||||
|
r897 | |||
|
r576 | if not self.__execute(chk_command): | ||
if not self.__execute(mkdir_command): | ||||
self.remotefolder = None | ||||
return 0 | ||||
|
r897 | |||
|
r573 | self.remotefolder = remotefolder | ||
|
r897 | |||
|
r573 | return 1 | ||
|
r897 | |||
|
r573 | def sendFile(self, fullfilename): | ||
if not self.status: | ||||
return 0 | ||||
|
r897 | |||
|
r573 | try: | ||
self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder) | ||||
|
r653 | except scp.ScpError, e: | ||
print "[SSH Server]", str(e) | ||||
|
r573 | return 0 | ||
|
r897 | |||
|
r576 | remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1]) | ||
command = 'chmod 775 %s' %remotefile | ||||
|
r897 | |||
|
r576 | return self.__execute(command) | ||
|
r897 | |||
|
r573 | class SendToServer(ProcessingUnit): | ||
|
r897 | |||
def __init__(self, **kwargs): | ||||
ProcessingUnit.__init__(self, **kwargs) | ||||
|
r573 | self.isConfig = False | ||
|
r1026 | self.clientObj = None | ||
|
r573 | |||
|
r581 | def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs): | ||
|
r897 | |||
|
r573 | self.clientObj = None | ||
self.localfolder = localfolder | ||||
self.ext = ext | ||||
self.period = period | ||||
|
r897 | |||
|
r573 | if str.lower(protocol) == 'ftp': | ||
self.clientObj = FTPClient(server, username, password, remotefolder, period) | ||||
|
r897 | |||
|
r573 | if str.lower(protocol) == 'ssh': | ||
self.clientObj = SSHClient(server, username, password, remotefolder, period) | ||||
|
r897 | |||
|
r573 | if not self.clientObj: | ||
raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol | ||||
|
r897 | |||
|
r573 | self.clientObj.start() | ||
|
r897 | |||
|
r573 | def findFiles(self): | ||
|
r897 | |||
|
r606 | if not type(self.localfolder) == list: | ||
folderList = [self.localfolder] | ||||
else: | ||||
folderList = self.localfolder | ||||
|
r897 | |||
|
r633 | #Remove duplicate items | ||
folderList = list(set(folderList)) | ||||
|
r897 | |||
|
r606 | fullfilenameList = [] | ||
|
r897 | |||
|
r606 | for thisFolder in folderList: | ||
|
r897 | |||
|
r633 | print "[Remote Server]: Searching files on %s" %thisFolder | ||
|
r897 | |||
|
r606 | filenameList = glob.glob1(thisFolder, '*%s' %self.ext) | ||
|
r897 | |||
|
r606 | if len(filenameList) < 1: | ||
|
r1019 | |||
|
r606 | continue | ||
|
r897 | |||
|
r606 | for thisFile in filenameList: | ||
fullfilename = os.path.join(thisFolder, thisFile) | ||||
|
r897 | |||
|
r633 | if fullfilename in fullfilenameList: | ||
continue | ||||
|
r897 | |||
|
r633 | #Only files modified in the last 30 minutes are considered | ||
if os.path.getmtime(fullfilename) < time.time() - 30*60: | ||||
continue | ||||
|
r897 | |||
|
r606 | fullfilenameList.append(fullfilename) | ||
|
r897 | |||
|
r573 | return fullfilenameList | ||
|
r897 | |||
|
r573 | def run(self, **kwargs): | ||
if not self.isConfig: | ||||
self.init = time.time() | ||||
self.setup(**kwargs) | ||||
self.isConfig = True | ||||
|
r1011 | if not self.clientObj.is_alive(): | ||
print "[Remote Server]: Restarting connection " | ||||
self.setup(**kwargs) | ||||
|
r573 | if time.time() - self.init >= self.period: | ||
fullfilenameList = self.findFiles() | ||||
|
r897 | |||
|
r606 | if self.clientObj.updateFileList(fullfilenameList): | ||
print "[Remote Server]: Sending the next files ", str(fullfilenameList) | ||||
|
r573 | self.init = time.time() | ||
|
r897 | |||
|
r573 | def close(self): | ||
print "[Remote Server] Stopping thread" | ||||
self.clientObj.stop() | ||||
|
r897 | |||
|
r517 | class FTP(object): | ||
|
r488 | """ | ||
Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module | ||||
|
r897 | |||
|
r488 | Non-standard Python modules used: None | ||
|
r897 | |||
|
r488 | Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010 | ||
""" | ||||
|
r897 | |||
def __init__(self,server = None, username=None, password=None, remotefolder=None): | ||||
|
r488 | """ | ||
This method is used to setting parameters for FTP and establishing connection to remote server | ||||
|
r897 | |||
|
r488 | Inputs: | ||
|
r897 | server - remote server IP Address | ||
username - remote server Username | ||||
|
r488 | password - remote server password | ||
|
r897 | |||
|
r488 | remotefolder - remote server current working directory | ||
|
r897 | |||
|
r488 | Return: void | ||
|
r897 | |||
Affects: | ||||
|
r488 | self.status - in Error Case or Connection Failed this parameter is set to 1 else 0 | ||
|
r897 | |||
|
r488 | self.folderList - sub-folder list of remote folder | ||
|
r897 | |||
|
r488 | self.fileList - file list of remote folder | ||
|
r897 | |||
|
r488 | """ | ||
|
r897 | |||
|
r488 | if ((server == None) and (username==None) and (password==None) and (remotefolder==None)): | ||
server, username, password, remotefolder = self.parmsByDefault() | ||||
|
r897 | |||
|
r488 | self.server = server | ||
self.username = username | ||||
self.password = password | ||||
self.remotefolder = remotefolder | ||||
self.file = None | ||||
self.ftp = None | ||||
self.status = 0 | ||||
|
r897 | |||
|
r488 | try: | ||
self.ftp = ftplib.FTP(self.server) | ||||
self.ftp.login(self.username,self.password) | ||||
self.ftp.cwd(self.remotefolder) | ||||
|
r1034 | # print 'Connect to FTP Server: Successfully' | ||
|
r488 | |||
except ftplib.all_errors: | ||||
print 'Error FTP Service' | ||||
self.status = 1 | ||||
return | ||||
|
r897 | |||
|
r488 | self.dirList = [] | ||
try: | ||||
self.dirList = self.ftp.nlst() | ||||
|
r897 | |||
|
r488 | except ftplib.error_perm, resp: | ||
if str(resp) == "550 No files found": | ||||
print "no files in this directory" | ||||
self.status = 1 | ||||
return | ||||
|
r897 | |||
|
r488 | except ftplib.all_errors: | ||
print 'Error Displaying Dir-Files' | ||||
self.status = 1 | ||||
return | ||||
|
r897 | |||
|
r488 | self.fileList = [] | ||
self.folderList = [] | ||||
#only for test | ||||
for f in self.dirList: | ||||
name, ext = os.path.splitext(f) | ||||
if ext != '': | ||||
self.fileList.append(f) | ||||
|
r1026 | # print 'filename: %s - size: %d'%(f,self.ftp.size(f)) | ||
|
r488 | |||
def parmsByDefault(self): | ||||
server = 'jro-app.igp.gob.pe' | ||||
username = 'wmaster' | ||||
password = 'mst2010vhf' | ||||
remotefolder = '/home/wmaster/graficos' | ||||
|
r897 | |||
|
r488 | return server, username, password, remotefolder | ||
|
r897 | |||
|
r488 | def mkd(self,dirname): | ||
""" | ||||
mkd is used to make directory in remote server | ||||
|
r897 | |||
|
r488 | Input: | ||
dirname - directory name | ||||
|
r897 | |||
|
r488 | Return: | ||
1 in error case else 0 | ||||
""" | ||||
|
r897 | try: | ||
|
r488 | self.ftp.mkd(dirname) | ||
except: | ||||
print 'Error creating remote folder:%s'%dirname | ||||
return 1 | ||||
|
r897 | |||
|
r488 | return 0 | ||
|
r897 | |||
|
r488 | def delete(self,filename): | ||
""" | ||||
delete is used to delete file in current working directory of remote server | ||||
|
r897 | |||
|
r488 | Input: | ||
filename - filename to delete in remote folder | ||||
|
r897 | |||
|
r488 | Return: | ||
1 in error case else 0 | ||||
""" | ||||
|
r897 | |||
|
r488 | try: | ||
self.ftp.delete(filename) | ||||
except: | ||||
print 'Error deleting remote file:%s'%filename | ||||
return 1 | ||||
|
r897 | |||
|
r488 | return 0 | ||
|
r897 | |||
|
r488 | def download(self,filename,localfolder): | ||
""" | ||||
download is used to downloading file from remote folder into local folder | ||||
|
r897 | |||
|
r488 | Inputs: | ||
filename - filename to donwload | ||||
|
r897 | |||
|
r488 | localfolder - directory local to store filename | ||
|
r897 | |||
|
r488 | Returns: | ||
self.status - 1 in error case else 0 | ||||
""" | ||||
|
r897 | |||
|
r488 | self.status = 0 | ||
|
r897 | |||
|
r488 | if not(filename in self.fileList): | ||
print 'filename:%s not exists'%filename | ||||
self.status = 1 | ||||
return self.status | ||||
|
r897 | |||
|
r488 | newfilename = os.path.join(localfolder,filename) | ||
|
r897 | |||
self.file = open(newfilename, 'wb') | ||||
|
r488 | try: | ||
|
r897 | print 'Download: ' + filename | ||
|
r488 | self.ftp.retrbinary('RETR ' + filename, self.__handleDownload) | ||
print 'Download Complete' | ||||
except ftplib.all_errors: | ||||
print 'Error Downloading ' + filename | ||||
self.status = 1 | ||||
return self.status | ||||
|
r897 | |||
|
r488 | self.file.close() | ||
|
r897 | |||
return self.status | ||||
|
r488 | |||
def __handleDownload(self,block): | ||||
""" | ||||
__handleDownload is used to handle writing file | ||||
""" | ||||
self.file.write(block) | ||||
|
r897 | |||
|
r488 | def upload(self,filename,remotefolder=None): | ||
""" | ||||
upload is used to uploading local file to remote directory | ||||
|
r897 | |||
|
r488 | Inputs: | ||
filename - full path name of local file to store in remote directory | ||||
|
r897 | |||
remotefolder - remote directory | ||||
|
r488 | Returns: | ||
self.status - 1 in error case else 0 | ||||
""" | ||||
|
r897 | |||
|
r488 | if remotefolder == None: | ||
remotefolder = self.remotefolder | ||||
|
r897 | |||
|
r488 | self.status = 0 | ||
|
r897 | |||
|
r488 | try: | ||
|
r897 | self.ftp.cwd(remotefolder) | ||
|
r488 | self.file = open(filename, 'rb') | ||
|
r897 | |||
|
r488 | (head, tail) = os.path.split(filename) | ||
|
r897 | |||
|
r488 | command = "STOR " + tail | ||
|
r897 | |||
|
r488 | print 'Uploading: ' + tail | ||
|
r897 | self.ftp.storbinary(command, self.file) | ||
|
r488 | print 'Upload Completed' | ||
|
r897 | |||
|
r488 | except ftplib.all_errors: | ||
print 'Error Uploading ' + tail | ||||
self.status = 1 | ||||
return self.status | ||||
|
r897 | |||
|
r488 | self.file.close() | ||
|
r897 | |||
|
r488 | #back to initial directory in __init__() | ||
|
r897 | self.ftp.cwd(self.remotefolder) | ||
return self.status | ||||
|
r488 | def dir(self,remotefolder): | ||
""" | ||||
dir is used to change working directory of remote server and get folder and file list | ||||
|
r897 | |||
|
r488 | Input: | ||
remotefolder - current working directory | ||||
|
r897 | |||
|
r488 | Affects: | ||
self.fileList - file list of working directory | ||||
|
r897 | |||
Return: | ||||
|
r488 | infoList - list with filenames and size of file in bytes | ||
|
r897 | |||
|
r488 | self.folderList - folder list | ||
""" | ||||
|
r897 | |||
|
r488 | self.remotefolder = remotefolder | ||
print 'Change to ' + self.remotefolder | ||||
|
r897 | try: | ||
self.ftp.cwd(remotefolder) | ||||
|
r488 | except ftplib.all_errors: | ||
|
r897 | print 'Error Change to ' + self.remotefolder | ||
|
r488 | infoList = None | ||
self.folderList = None | ||||
return infoList,self.folderList | ||||
|
r897 | |||
|
r488 | self.dirList = [] | ||
try: | ||||
self.dirList = self.ftp.nlst() | ||||
|
r897 | |||
|
r488 | except ftplib.error_perm, resp: | ||
|
r897 | if str(resp) == "550 No files found": | ||
|
r488 | print "no files in this directory" | ||
infoList = None | ||||
self.folderList = None | ||||
return infoList,self.folderList | ||||
except ftplib.all_errors: | ||||
print 'Error Displaying Dir-Files' | ||||
infoList = None | ||||
self.folderList = None | ||||
|
r897 | return infoList,self.folderList | ||
infoList = [] | ||||
|
r488 | self.fileList = [] | ||
self.folderList = [] | ||||
for f in self.dirList: | ||||
name,ext = os.path.splitext(f) | ||||
|
r897 | if ext != '': | ||
|
r488 | self.fileList.append(f) | ||
value = (f,self.ftp.size(f)) | ||||
|
r897 | infoList.append(value) | ||
if ext == '': | ||||
|
r488 | self.folderList.append(f) | ||
|
r897 | |||
|
r488 | return infoList,self.folderList | ||
|
r897 | |||
|
r488 | def close(self): | ||
""" | ||||
close is used to close and end FTP connection | ||||
|
r897 | |||
|
r488 | Inputs: None | ||
|
r897 | |||
|
r488 | Return: void | ||
|
r897 | |||
|
r488 | """ | ||
self.ftp.close() | ||||
class SendByFTP(Operation): | ||||
|
r897 | |||
def __init__(self, **kwargs): | ||||
Operation.__init__(self, **kwargs) | ||||
|
r488 | self.status = 1 | ||
|
r489 | self.counter = 0 | ||
|
r897 | |||
|
r488 | def error_print(self, ValueError): | ||
|
r897 | |||
|
r488 | print ValueError, 'Error FTP' | ||
print "don't worry the program is running..." | ||||
|
r897 | |||
|
r517 | def worker_ftp(self, server, username, password, remotefolder, filenameList): | ||
|
r897 | |||
|
r573 | self.ftpClientObj = FTP(server, username, password, remotefolder) | ||
|
r517 | for filename in filenameList: | ||
|
r573 | self.ftpClientObj.upload(filename) | ||
self.ftpClientObj.close() | ||||
|
r897 | |||
|
r517 | def ftp_thread(self, server, username, password, remotefolder): | ||
|
r488 | if not(self.status): | ||
return | ||||
|
r897 | |||
|
r633 | import multiprocessing | ||
|
r897 | |||
|
r517 | p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,)) | ||
p.start() | ||||
|
r897 | |||
|
r517 | p.join(3) | ||
|
r897 | |||
|
r517 | if p.is_alive(): | ||
p.terminate() | ||||
p.join() | ||||
print 'killing ftp process...' | ||||
|
r488 | self.status = 0 | ||
return | ||||
|
r897 | |||
|
r517 | self.status = 1 | ||
return | ||||
|
r897 | |||
|
r488 | def filterByExt(self, ext, localfolder): | ||
fnameList = glob.glob1(localfolder,ext) | ||||
self.filenameList = [os.path.join(localfolder,x) for x in fnameList] | ||||
if len(self.filenameList) == 0: | ||||
self.status = 0 | ||||
|
r897 | |||
|
r489 | def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1): | ||
|
r897 | |||
self.counter += 1 | ||||
|
r489 | if self.counter >= period: | ||
self.filterByExt(ext, localfolder) | ||||
|
r897 | |||
|
r517 | self.ftp_thread(server, username, password, remotefolder) | ||
|
r489 | self.counter = 0 | ||
|
r897 | |||
|
r517 | self.status = 1 | ||