jroutils_ftp.py
1008 lines
| 28.8 KiB
| text/x-python
|
PythonLexer
Daniel Valdez
|
r488 | ''' | ||
| @author: Daniel Suarez | ||||
| ''' | ||||
| import os | ||||
| import glob | ||||
| import ftplib | ||||
Miguel Valdez
|
r573 | |||
| try: | ||||
| import paramiko | ||||
| import scp | ||||
| except: | ||||
|
Miguel Valdez
|
r641 | print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server" | ||
|
Miguel Valdez
|
r573 | |||
| import time | ||||
|
Miguel Valdez
|
r633 | import threading | ||
| Thread = threading.Thread | ||||
| # try: | ||||
| # from gevent import sleep | ||||
| # except: | ||||
| from time import sleep | ||||
|
Miguel Valdez
|
r573 | |||
|
Miguel Valdez
|
r568 | from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation | ||
|
Daniel Valdez
|
r488 | |||
|
Miguel Valdez
|
r633 | class Remote(Thread): | ||
|
Miguel Valdez
|
r573 | """ | ||
| Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are | ||||
| used to upload or download files remotely. | ||||
| Non-standard Python modules used: | ||||
| None | ||||
| Written by: | ||||
José Chávez
|
r1026 | "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015 | ||
|
Miguel Valdez
|
r573 | """ | ||
| server = None | ||||
| username = None | ||||
| password = None | ||||
| remotefolder = None | ||||
| period = 60 | ||||
| fileList = [] | ||||
| bussy = False | ||||
| def __init__(self, server, username, password, remotefolder, period=60): | ||||
|
Miguel Valdez
|
r633 | Thread.__init__(self) | ||
|
Miguel Valdez
|
r573 | |||
|
Miguel Valdez
|
r606 | self.setDaemon(True) | ||
|
Miguel Valdez
|
r573 | self.status = 0 | ||
|
Miguel Valdez
|
r653 | self.__server = server | ||
| self.__username = username | ||||
| self.__password = password | ||||
| self.__remotefolder = remotefolder | ||||
|
Miguel Valdez
|
r573 | self.period = period | ||
|
Miguel Valdez
|
r653 | |||
|
Miguel Valdez
|
r573 | self.fileList = [] | ||
| self.bussy = False | ||||
| self.stopFlag = False | ||||
|
Miguel Valdez
|
r653 | print "[Remote Server] Opening server: %s" %self.__server | ||
| if self.open(self.__server, self.__username, self.__password, self.__remotefolder): | ||||
| print "[Remote Server] %s server was opened successfully" %self.__server | ||||
| self.close() | ||||
|
Miguel Valdez
|
r633 | self.mutex = threading.Lock() | ||
|
Miguel Valdez
|
r573 | def stop(self): | ||
| self.stopFlag = True | ||||
|
Miguel Valdez
|
r633 | self.join(10) | ||
|
Miguel Valdez
|
r573 | |||
| def open(self): | ||||
| """ | ||||
| Connect to server and create a connection class (FTP or SSH) to remote server. | ||||
| """ | ||||
| raise NotImplementedError, "Implement this method in child class" | ||||
| def close(self): | ||||
| """ | ||||
| Close connection to server | ||||
| """ | ||||
| raise NotImplementedError, "Implement this method in child class" | ||||
| def mkdir(self, remotefolder): | ||||
| """ | ||||
| Create a folder remotely | ||||
| """ | ||||
| raise NotImplementedError, "Implement this method in child class" | ||||
| def cd(self, remotefolder): | ||||
| """ | ||||
| Change working directory in remote server | ||||
| """ | ||||
| raise NotImplementedError, "Implement this method in child class" | ||||
| def download(self, filename, localfolder=None): | ||||
| """ | ||||
| Download a file from server to local host | ||||
| """ | ||||
| raise NotImplementedError, "Implement this method in child class" | ||||
| def sendFile(self, fullfilename): | ||||
| """ | ||||
| sendFile method is used to upload a local file to the current directory in remote server | ||||
| Inputs: | ||||
| fullfilename - full path name of local file to store in remote directory | ||||
| Returns: | ||||
| 0 in error case else 1 | ||||
| """ | ||||
| raise NotImplementedError, "Implement this method in child class" | ||||
| def upload(self, fullfilename, remotefolder=None): | ||||
| """ | ||||
| upload method is used to upload a local file to remote directory. This method changes | ||||
| working directory before sending a file. | ||||
| Inputs: | ||||
| fullfilename - full path name of local file to store in remote directory | ||||
| remotefolder - remote directory | ||||
| Returns: | ||||
| 0 in error case else 1 | ||||
| """ | ||||
| print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder) | ||||
| if not self.status: | ||||
| return 0 | ||||
| if remotefolder == None: | ||||
| remotefolder = self.remotefolder | ||||
| if not self.cd(remotefolder): | ||||
| return 0 | ||||
| if not self.sendFile(fullfilename): | ||||
| print "[Remote Server] Error uploading file %s" %fullfilename | ||||
| return 0 | ||||
| print "[Remote Server] upload finished successfully" | ||||
| return 1 | ||||
| def delete(self, filename): | ||||
| """ | ||||
| Remove a file from remote server | ||||
| """ | ||||
| pass | ||||
| def updateFileList(self, fileList): | ||||
| """ | ||||
| Remove a file from remote server | ||||
| """ | ||||
| if fileList == self.fileList: | ||||
|
Miguel Valdez
|
r606 | return 0 | ||
|
Miguel Valdez
|
r573 | |||
|
Miguel Valdez
|
r633 | self.mutex.acquire() | ||
|
José Chávez
|
r1026 | # init = time.time() | ||
| # | ||||
| # while(self.bussy): | ||||
| # sleep(0.1) | ||||
| # if time.time() - init > 2*self.period: | ||||
| # return 0 | ||||
|
Miguel Valdez
|
r573 | self.fileList = fileList | ||
|
Miguel Valdez
|
r633 | self.mutex.release() | ||
|
Miguel Valdez
|
r573 | return 1 | ||
| def run(self): | ||||
|
Miguel Valdez
|
r580 | if not self.status: | ||
| print "Finishing FTP service" | ||||
| return | ||||
|
Miguel Valdez
|
r573 | if not self.cd(self.remotefolder): | ||
|
Miguel Valdez
|
r576 | raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder | ||
|
Miguel Valdez
|
r573 | |||
| while True: | ||||
|
Miguel Valdez
|
r633 | for i in range(self.period): | ||
| if self.stopFlag: | ||||
| break | ||||
| sleep(1) | ||||
|
Miguel Valdez
|
r573 | |||
|
Miguel Valdez
|
r633 | if self.stopFlag: | ||
| break | ||||
|
José Chávez
|
r1026 | # self.bussy = True | ||
|
Miguel Valdez
|
r633 | self.mutex.acquire() | ||
|
Miguel Valdez
|
r573 | |||
|
Miguel Valdez
|
r653 | print "[Remote Server] Opening %s" %self.__server | ||
| if not self.open(self.__server, self.__username, self.__password, self.__remotefolder): | ||||
|
Miguel Valdez
|
r661 | self.mutex.release() | ||
|
Miguel Valdez
|
r662 | continue | ||
|
Miguel Valdez
|
r653 | |||
|
Miguel Valdez
|
r573 | for thisFile in self.fileList: | ||
|
Miguel Valdez
|
r646 | self.upload(thisFile, self.remotefolder) | ||
|
Miguel Valdez
|
r573 | |||
|
Miguel Valdez
|
r653 | print "[Remote Server] Closing %s" %self.__server | ||
| self.close() | ||||
|
Miguel Valdez
|
r633 | self.mutex.release() | ||
| # self.bussy = False | ||||
|
Miguel Valdez
|
r606 | |||
|
Miguel Valdez
|
r573 | print "[Remote Server] Thread stopped successfully" | ||
| class FTPClient(Remote): | ||||
| __ftpClientObj = None | ||||
| def __init__(self, server, username, password, remotefolder, period=60): | ||||
| """ | ||||
| """ | ||||
| Remote.__init__(self, server, username, password, remotefolder, period) | ||||
| def open(self, server, username, password, remotefolder): | ||||
| """ | ||||
| This method is used to set FTP parameters and establish a connection to remote server | ||||
| Inputs: | ||||
| server - remote server IP Address | ||||
| username - remote server Username | ||||
| password - remote server password | ||||
| remotefolder - remote server current working directory | ||||
|
Miguel Valdez
|
r653 | Return: | ||
| Boolean - Returns 1 if a connection has been established, 0 otherwise | ||||
|
Miguel Valdez
|
r573 | |||
| Affects: | ||||
| self.status - in case of error or fail connection this parameter is set to 0 else 1 | ||||
| """ | ||||
| if server == None: | ||||
| raise ValueError, "FTP server should be defined" | ||||
| if username == None: | ||||
| raise ValueError, "FTP username should be defined" | ||||
| if password == None: | ||||
| raise ValueError, "FTP password should be defined" | ||||
| if remotefolder == None: | ||||
| raise ValueError, "FTP remote folder should be defined" | ||||
| try: | ||||
| ftpClientObj = ftplib.FTP(server) | ||||
|
Miguel Valdez
|
r653 | except ftplib.all_errors, e: | ||
| print "[FTP Server]: FTP server connection fail: %s" %server | ||||
| print "[FTP Server]:", e | ||||
|
Miguel Valdez
|
r573 | self.status = 0 | ||
| return 0 | ||||
| try: | ||||
| ftpClientObj.login(username, password) | ||||
| except ftplib.all_errors: | ||||
|
Miguel Valdez
|
r653 | print "[FTP Server]: FTP username or password are incorrect" | ||
|
Miguel Valdez
|
r573 | self.status = 0 | ||
| return 0 | ||||
| if remotefolder == None: | ||||
| remotefolder = ftpClientObj.pwd() | ||||
| else: | ||||
| try: | ||||
| ftpClientObj.cwd(remotefolder) | ||||
| except ftplib.all_errors: | ||||
|
Miguel Valdez
|
r653 | print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder | ||
|
Miguel Valdez
|
r573 | remotefolder = ftpClientObj.pwd() | ||
| self.server = server | ||||
| self.username = username | ||||
| self.password = password | ||||
| self.remotefolder = remotefolder | ||||
| self.__ftpClientObj = ftpClientObj | ||||
| self.status = 1 | ||||
| return 1 | ||||
| def close(self): | ||||
| """ | ||||
| Close connection to remote server | ||||
| """ | ||||
| if not self.status: | ||||
| return 0 | ||||
| self.__ftpClientObj.close() | ||||
| def mkdir(self, remotefolder): | ||||
| """ | ||||
| mkdir is used to make a new directory in remote server | ||||
| Input: | ||||
| remotefolder - directory name | ||||
| Return: | ||||
| 0 in error case else 1 | ||||
| """ | ||||
| if not self.status: | ||||
| return 0 | ||||
| try: | ||||
| self.__ftpClientObj.mkd(dirname) | ||||
| except ftplib.all_errors: | ||||
|
Miguel Valdez
|
r653 | print "[FTP Server]: Error creating remote folder: %s" %remotefolder | ||
|
Miguel Valdez
|
r573 | return 0 | ||
| return 1 | ||||
| def cd(self, remotefolder): | ||||
| """ | ||||
| cd is used to change remote working directory on server | ||||
| Input: | ||||
| remotefolder - current working directory | ||||
| Affects: | ||||
| self.remotefolder | ||||
| Return: | ||||
| 0 in case of error else 1 | ||||
| """ | ||||
| if not self.status: | ||||
| return 0 | ||||
| if remotefolder == self.remotefolder: | ||||
| return 1 | ||||
| try: | ||||
| self.__ftpClientObj.cwd(remotefolder) | ||||
| except ftplib.all_errors: | ||||
|
Miguel Valdez
|
r653 | print '[FTP Server]: Error changing to %s' %remotefolder | ||
| print '[FTP Server]: Trying to create remote folder' | ||||
|
Miguel Valdez
|
r573 | |||
| if not self.mkdir(remotefolder): | ||||
|
Miguel Valdez
|
r653 | print '[FTP Server]: Remote folder could not be created' | ||
|
Miguel Valdez
|
r573 | return 0 | ||
| try: | ||||
| self.__ftpClientObj.cwd(remotefolder) | ||||
| except ftplib.all_errors: | ||||
| return 0 | ||||
| self.remotefolder = remotefolder | ||||
| return 1 | ||||
| def sendFile(self, fullfilename): | ||||
| if not self.status: | ||||
| return 0 | ||||
|
Miguel Valdez
|
r653 | fp = open(fullfilename, 'rb') | ||
|
Miguel Valdez
|
r573 | |||
|
Miguel Valdez
|
r606 | filename = os.path.basename(fullfilename) | ||
|
Miguel Valdez
|
r573 | |||
| command = "STOR %s" %filename | ||||
| try: | ||||
|
Miguel Valdez
|
r653 | self.__ftpClientObj.storbinary(command, fp) | ||
| except ftplib.all_errors, e: | ||||
| print "[FTP Server]:", e | ||||
|
Miguel Valdez
|
r573 | return 0 | ||
| try: | ||||
| self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename) | ||||
| except ftplib.all_errors, e: | ||||
|
Miguel Valdez
|
r653 | print "[FTP Server]:", e | ||
|
Miguel Valdez
|
r573 | |||
|
Miguel Valdez
|
r653 | fp.close() | ||
|
Miguel Valdez
|
r573 | |||
| return 1 | ||||
| class SSHClient(Remote): | ||||
| __sshClientObj = None | ||||
| __scpClientObj = None | ||||
| def __init__(self, server, username, password, remotefolder, period=60): | ||||
| """ | ||||
| """ | ||||
| Remote.__init__(self, server, username, password, remotefolder, period) | ||||
| def open(self, server, username, password, remotefolder, port=22): | ||||
| """ | ||||
|
José Chávez
|
r1026 | This method is used to set SSH parameters and establish a connection to a remote server | ||
|
Miguel Valdez
|
r573 | |||
|
José Chávez
|
r1026 | Inputs: | ||
| server - remote server IP Address | ||||
| username - remote server Username | ||||
| password - remote server password | ||||
| remotefolder - remote server current working directory | ||||
|
Miguel Valdez
|
r573 | |||
|
José Chávez
|
r1026 | Return: void | ||
|
Miguel Valdez
|
r573 | |||
|
José Chávez
|
r1026 | Affects: | ||
| self.status - in case of error or fail connection this parameter is set to 0 else 1 | ||||
|
Miguel Valdez
|
r573 | |||
| """ | ||||
|
Miguel Valdez
|
r653 | import socket | ||
|
Miguel Valdez
|
r573 | if server == None: | ||
| raise ValueError, "SSH server should be defined" | ||||
| if username == None: | ||||
| raise ValueError, "SSH username should be defined" | ||||
| if password == None: | ||||
| raise ValueError, "SSH password should be defined" | ||||
| if remotefolder == None: | ||||
| raise ValueError, "SSH remote folder should be defined" | ||||
|
Miguel Valdez
|
r653 | sshClientObj = paramiko.SSHClient() | ||
|
Miguel Valdez
|
r573 | |||
| sshClientObj.load_system_host_keys() | ||||
| sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy()) | ||||
|
Miguel Valdez
|
r653 | self.status = 0 | ||
|
Miguel Valdez
|
r573 | try: | ||
| sshClientObj.connect(server, username=username, password=password, port=port) | ||||
|
Miguel Valdez
|
r653 | except paramiko.AuthenticationException, e: | ||
|
José Chávez
|
r1026 | # print "SSH username or password are incorrect: %s" | ||
|
Miguel Valdez
|
r653 | print "[SSH Server]:", e | ||
| return 0 | ||||
| except SSHException, e: | ||||
| print "[SSH Server]:", e | ||||
| return 0 | ||||
| except socket.error: | ||||
|
Miguel Valdez
|
r573 | self.status = 0 | ||
|
Miguel Valdez
|
r653 | print "[SSH Server]:", e | ||
|
Miguel Valdez
|
r573 | return 0 | ||
|
Miguel Valdez
|
r653 | self.status = 1 | ||
|
Miguel Valdez
|
r573 | scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30) | ||
| if remotefolder == None: | ||||
| remotefolder = self.pwd() | ||||
| self.server = server | ||||
| self.username = username | ||||
| self.password = password | ||||
| self.__sshClientObj = sshClientObj | ||||
| self.__scpClientObj = scpClientObj | ||||
| self.status = 1 | ||||
|
Miguel Valdez
|
r576 | if not self.cd(remotefolder): | ||
|
Miguel Valdez
|
r653 | raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder | ||
|
Miguel Valdez
|
r576 | return 0 | ||
| self.remotefolder = remotefolder | ||||
|
Miguel Valdez
|
r573 | return 1 | ||
| def close(self): | ||||
| """ | ||||
|
José Chávez
|
r1026 | Close connection to remote server | ||
|
Miguel Valdez
|
r573 | """ | ||
| if not self.status: | ||||
| return 0 | ||||
|
Miguel Valdez
|
r653 | self.__scpClientObj.close() | ||
| self.__sshClientObj.close() | ||||
|
Miguel Valdez
|
r576 | |||
| def __execute(self, command): | ||||
|
Miguel Valdez
|
r573 | """ | ||
|
José Chávez
|
r1026 | __execute a command on remote server | ||
| Input: | ||||
| command - Exmaple 'ls -l' | ||||
| Return: | ||||
| 0 in error case else 1 | ||||
|
Miguel Valdez
|
r573 | """ | ||
| if not self.status: | ||||
| return 0 | ||||
|
Miguel Valdez
|
r576 | |||
| stdin, stdout, stderr = self.__sshClientObj.exec_command(command) | ||||
|
Miguel Valdez
|
r573 | |||
|
Miguel Valdez
|
r576 | result = stderr.readlines() | ||
|
Miguel Valdez
|
r573 | if len(result) > 1: | ||
| return 0 | ||||
|
Miguel Valdez
|
r576 | |||
| result = stdout.readlines() | ||||
| if len(result) > 1: | ||||
| return result[0][:-1] | ||||
|
Miguel Valdez
|
r573 | |||
| return 1 | ||||
|
Miguel Valdez
|
r576 | |||
| def mkdir(self, remotefolder): | ||||
| """ | ||||
|
José Chávez
|
r1026 | mkdir is used to make a new directory in remote server | ||
| Input: | ||||
| remotefolder - directory name | ||||
| Return: | ||||
| 0 in error case else 1 | ||||
|
Miguel Valdez
|
r576 | """ | ||
| command = 'mkdir %s' %remotefolder | ||||
| return self.__execute(command) | ||||
|
Miguel Valdez
|
r573 | |||
| def pwd(self): | ||||
|
Miguel Valdez
|
r576 | command = 'pwd' | ||
|
Miguel Valdez
|
r573 | |||
|
Miguel Valdez
|
r576 | return self.__execute(command) | ||
|
Miguel Valdez
|
r573 | |||
| def cd(self, remotefolder): | ||||
| """ | ||||
|
José Chávez
|
r1026 | cd is used to change remote working directory on server | ||
|
Miguel Valdez
|
r573 | |||
|
José Chávez
|
r1026 | Input: | ||
| remotefolder - current working directory | ||||
| Affects: | ||||
| self.remotefolder | ||||
| Return: | ||||
| 0 in case of error else 1 | ||||
|
Miguel Valdez
|
r573 | """ | ||
| if not self.status: | ||||
| return 0 | ||||
|
Miguel Valdez
|
r576 | |||
|
Miguel Valdez
|
r573 | if remotefolder == self.remotefolder: | ||
| return 1 | ||||
|
Miguel Valdez
|
r576 | |||
| chk_command = "cd %s; pwd" %remotefolder | ||||
| mkdir_command = "mkdir %s" %remotefolder | ||||
| if not self.__execute(chk_command): | ||||
| if not self.__execute(mkdir_command): | ||||
| self.remotefolder = None | ||||
| return 0 | ||||
|
Miguel Valdez
|
r573 | |||
| self.remotefolder = remotefolder | ||||
| return 1 | ||||
| def sendFile(self, fullfilename): | ||||
| if not self.status: | ||||
| return 0 | ||||
| try: | ||||
| self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder) | ||||
|
Miguel Valdez
|
r653 | except scp.ScpError, e: | ||
| print "[SSH Server]", str(e) | ||||
|
Miguel Valdez
|
r573 | return 0 | ||
|
Miguel Valdez
|
r576 | remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1]) | ||
| command = 'chmod 775 %s' %remotefile | ||||
| return self.__execute(command) | ||||
|
Miguel Valdez
|
r573 | |||
| class SendToServer(ProcessingUnit): | ||||
| def __init__(self): | ||||
| ProcessingUnit.__init__(self) | ||||
| self.isConfig = False | ||||
|
José Chávez
|
r1026 | self.clientObj = None | ||
|
Miguel Valdez
|
r573 | |||
|
Miguel Valdez
|
r581 | def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs): | ||
|
Miguel Valdez
|
r573 | |||
| self.clientObj = None | ||||
| self.localfolder = localfolder | ||||
| self.ext = ext | ||||
| self.period = period | ||||
| if str.lower(protocol) == 'ftp': | ||||
| self.clientObj = FTPClient(server, username, password, remotefolder, period) | ||||
| if str.lower(protocol) == 'ssh': | ||||
| self.clientObj = SSHClient(server, username, password, remotefolder, period) | ||||
| if not self.clientObj: | ||||
| raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol | ||||
| self.clientObj.start() | ||||
| def findFiles(self): | ||||
|
Miguel Valdez
|
r606 | if not type(self.localfolder) == list: | ||
| folderList = [self.localfolder] | ||||
| else: | ||||
| folderList = self.localfolder | ||||
|
Miguel Valdez
|
r573 | |||
|
Miguel Valdez
|
r633 | #Remove duplicate items | ||
| folderList = list(set(folderList)) | ||||
|
Miguel Valdez
|
r606 | fullfilenameList = [] | ||
| for thisFolder in folderList: | ||||
|
Miguel Valdez
|
r633 | |||
| print "[Remote Server]: Searching files on %s" %thisFolder | ||||
|
Miguel Valdez
|
r606 | filenameList = glob.glob1(thisFolder, '*%s' %self.ext) | ||
| if len(filenameList) < 1: | ||||
| continue | ||||
| for thisFile in filenameList: | ||||
| fullfilename = os.path.join(thisFolder, thisFile) | ||||
|
Miguel Valdez
|
r633 | |||
| if fullfilename in fullfilenameList: | ||||
| continue | ||||
| #Only files modified in the last 30 minutes are considered | ||||
| if os.path.getmtime(fullfilename) < time.time() - 30*60: | ||||
| continue | ||||
|
Miguel Valdez
|
r606 | fullfilenameList.append(fullfilename) | ||
|
Miguel Valdez
|
r573 | return fullfilenameList | ||
| def run(self, **kwargs): | ||||
| if not self.isConfig: | ||||
| self.init = time.time() | ||||
| self.setup(**kwargs) | ||||
| self.isConfig = True | ||||
|
José Chávez
|
r1011 | if not self.clientObj.is_alive(): | ||
| print "[Remote Server]: Restarting connection " | ||||
| self.setup(**kwargs) | ||||
|
Miguel Valdez
|
r573 | if time.time() - self.init >= self.period: | ||
| fullfilenameList = self.findFiles() | ||||
|
Miguel Valdez
|
r606 | |||
| if self.clientObj.updateFileList(fullfilenameList): | ||||
| print "[Remote Server]: Sending the next files ", str(fullfilenameList) | ||||
|
Miguel Valdez
|
r573 | self.init = time.time() | ||
| def close(self): | ||||
| print "[Remote Server] Stopping thread" | ||||
| self.clientObj.stop() | ||||
|
Daniel Valdez
|
r517 | class FTP(object): | ||
|
Daniel Valdez
|
r488 | """ | ||
| Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module | ||||
| Non-standard Python modules used: None | ||||
| Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010 | ||||
| """ | ||||
| def __init__(self,server = None, username=None, password=None, remotefolder=None): | ||||
| """ | ||||
| This method is used to setting parameters for FTP and establishing connection to remote server | ||||
| Inputs: | ||||
| server - remote server IP Address | ||||
| username - remote server Username | ||||
| password - remote server password | ||||
| remotefolder - remote server current working directory | ||||
| Return: void | ||||
| Affects: | ||||
| self.status - in Error Case or Connection Failed this parameter is set to 1 else 0 | ||||
| self.folderList - sub-folder list of remote folder | ||||
| self.fileList - file list of remote folder | ||||
| """ | ||||
| if ((server == None) and (username==None) and (password==None) and (remotefolder==None)): | ||||
| server, username, password, remotefolder = self.parmsByDefault() | ||||
| self.server = server | ||||
| self.username = username | ||||
| self.password = password | ||||
| self.remotefolder = remotefolder | ||||
| self.file = None | ||||
| self.ftp = None | ||||
| self.status = 0 | ||||
| try: | ||||
| self.ftp = ftplib.FTP(self.server) | ||||
| self.ftp.login(self.username,self.password) | ||||
| self.ftp.cwd(self.remotefolder) | ||||
|
José Chávez
|
r1026 | # print 'Connect to FTP Server: Successfully' | ||
|
Daniel Valdez
|
r488 | |||
| except ftplib.all_errors: | ||||
| print 'Error FTP Service' | ||||
| self.status = 1 | ||||
| return | ||||
| self.dirList = [] | ||||
| try: | ||||
| self.dirList = self.ftp.nlst() | ||||
| except ftplib.error_perm, resp: | ||||
| if str(resp) == "550 No files found": | ||||
| print "no files in this directory" | ||||
| self.status = 1 | ||||
| return | ||||
| except ftplib.all_errors: | ||||
| print 'Error Displaying Dir-Files' | ||||
| self.status = 1 | ||||
| return | ||||
| self.fileList = [] | ||||
| self.folderList = [] | ||||
| #only for test | ||||
| for f in self.dirList: | ||||
| name, ext = os.path.splitext(f) | ||||
| if ext != '': | ||||
| self.fileList.append(f) | ||||
|
José Chávez
|
r1026 | # print 'filename: %s - size: %d'%(f,self.ftp.size(f)) | ||
|
Daniel Valdez
|
r488 | |||
| def parmsByDefault(self): | ||||
| server = 'jro-app.igp.gob.pe' | ||||
| username = 'wmaster' | ||||
| password = 'mst2010vhf' | ||||
| remotefolder = '/home/wmaster/graficos' | ||||
| return server, username, password, remotefolder | ||||
| def mkd(self,dirname): | ||||
| """ | ||||
| mkd is used to make directory in remote server | ||||
| Input: | ||||
| dirname - directory name | ||||
| Return: | ||||
| 1 in error case else 0 | ||||
| """ | ||||
| try: | ||||
| self.ftp.mkd(dirname) | ||||
| except: | ||||
| print 'Error creating remote folder:%s'%dirname | ||||
| return 1 | ||||
| return 0 | ||||
| def delete(self,filename): | ||||
| """ | ||||
| delete is used to delete file in current working directory of remote server | ||||
| Input: | ||||
| filename - filename to delete in remote folder | ||||
| Return: | ||||
| 1 in error case else 0 | ||||
| """ | ||||
| try: | ||||
| self.ftp.delete(filename) | ||||
| except: | ||||
| print 'Error deleting remote file:%s'%filename | ||||
| return 1 | ||||
| return 0 | ||||
| def download(self,filename,localfolder): | ||||
| """ | ||||
| download is used to downloading file from remote folder into local folder | ||||
| Inputs: | ||||
| filename - filename to donwload | ||||
| localfolder - directory local to store filename | ||||
| Returns: | ||||
| self.status - 1 in error case else 0 | ||||
| """ | ||||
| self.status = 0 | ||||
| if not(filename in self.fileList): | ||||
| print 'filename:%s not exists'%filename | ||||
| self.status = 1 | ||||
| return self.status | ||||
| newfilename = os.path.join(localfolder,filename) | ||||
| self.file = open(newfilename, 'wb') | ||||
| try: | ||||
| print 'Download: ' + filename | ||||
| self.ftp.retrbinary('RETR ' + filename, self.__handleDownload) | ||||
| print 'Download Complete' | ||||
| except ftplib.all_errors: | ||||
| print 'Error Downloading ' + filename | ||||
| self.status = 1 | ||||
| return self.status | ||||
| self.file.close() | ||||
| return self.status | ||||
| def __handleDownload(self,block): | ||||
| """ | ||||
| __handleDownload is used to handle writing file | ||||
| """ | ||||
| self.file.write(block) | ||||
| def upload(self,filename,remotefolder=None): | ||||
| """ | ||||
| upload is used to uploading local file to remote directory | ||||
| Inputs: | ||||
| filename - full path name of local file to store in remote directory | ||||
| remotefolder - remote directory | ||||
| Returns: | ||||
| self.status - 1 in error case else 0 | ||||
| """ | ||||
| if remotefolder == None: | ||||
| remotefolder = self.remotefolder | ||||
| self.status = 0 | ||||
| try: | ||||
| self.ftp.cwd(remotefolder) | ||||
| self.file = open(filename, 'rb') | ||||
| (head, tail) = os.path.split(filename) | ||||
| command = "STOR " + tail | ||||
| print 'Uploading: ' + tail | ||||
| self.ftp.storbinary(command, self.file) | ||||
| print 'Upload Completed' | ||||
| except ftplib.all_errors: | ||||
| print 'Error Uploading ' + tail | ||||
| self.status = 1 | ||||
| return self.status | ||||
| self.file.close() | ||||
| #back to initial directory in __init__() | ||||
| self.ftp.cwd(self.remotefolder) | ||||
| return self.status | ||||
| def dir(self,remotefolder): | ||||
| """ | ||||
| dir is used to change working directory of remote server and get folder and file list | ||||
| Input: | ||||
| remotefolder - current working directory | ||||
| Affects: | ||||
| self.fileList - file list of working directory | ||||
| Return: | ||||
| infoList - list with filenames and size of file in bytes | ||||
| self.folderList - folder list | ||||
| """ | ||||
| self.remotefolder = remotefolder | ||||
| print 'Change to ' + self.remotefolder | ||||
| try: | ||||
| self.ftp.cwd(remotefolder) | ||||
| except ftplib.all_errors: | ||||
| print 'Error Change to ' + self.remotefolder | ||||
| infoList = None | ||||
| self.folderList = None | ||||
| return infoList,self.folderList | ||||
| self.dirList = [] | ||||
| try: | ||||
| self.dirList = self.ftp.nlst() | ||||
| except ftplib.error_perm, resp: | ||||
| if str(resp) == "550 No files found": | ||||
| print "no files in this directory" | ||||
| infoList = None | ||||
| self.folderList = None | ||||
| return infoList,self.folderList | ||||
| except ftplib.all_errors: | ||||
| print 'Error Displaying Dir-Files' | ||||
| infoList = None | ||||
| self.folderList = None | ||||
| return infoList,self.folderList | ||||
| infoList = [] | ||||
| self.fileList = [] | ||||
| self.folderList = [] | ||||
| for f in self.dirList: | ||||
| name,ext = os.path.splitext(f) | ||||
| if ext != '': | ||||
| self.fileList.append(f) | ||||
| value = (f,self.ftp.size(f)) | ||||
| infoList.append(value) | ||||
| if ext == '': | ||||
| self.folderList.append(f) | ||||
| return infoList,self.folderList | ||||
| def close(self): | ||||
| """ | ||||
| close is used to close and end FTP connection | ||||
| Inputs: None | ||||
| Return: void | ||||
| """ | ||||
| self.ftp.close() | ||||
| class SendByFTP(Operation): | ||||
|
Miguel Valdez
|
r573 | |||
|
Daniel Valdez
|
r488 | def __init__(self): | ||
|
Miguel Valdez
|
r573 | |||
|
Daniel Valdez
|
r488 | self.status = 1 | ||
|
Daniel Valdez
|
r489 | self.counter = 0 | ||
|
Daniel Valdez
|
r488 | |||
| def error_print(self, ValueError): | ||||
|
Miguel Valdez
|
r573 | |||
|
Daniel Valdez
|
r488 | print ValueError, 'Error FTP' | ||
| print "don't worry the program is running..." | ||||
|
Daniel Valdez
|
r517 | def worker_ftp(self, server, username, password, remotefolder, filenameList): | ||
|
Miguel Valdez
|
r573 | self.ftpClientObj = FTP(server, username, password, remotefolder) | ||
|
Daniel Valdez
|
r517 | for filename in filenameList: | ||
|
Miguel Valdez
|
r573 | self.ftpClientObj.upload(filename) | ||
| self.ftpClientObj.close() | ||||
|
Daniel Valdez
|
r488 | |||
|
Daniel Valdez
|
r517 | def ftp_thread(self, server, username, password, remotefolder): | ||
|
Daniel Valdez
|
r488 | if not(self.status): | ||
| return | ||||
|
Miguel Valdez
|
r633 | import multiprocessing | ||
|
Daniel Valdez
|
r517 | p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,)) | ||
| p.start() | ||||
| p.join(3) | ||||
| if p.is_alive(): | ||||
| p.terminate() | ||||
| p.join() | ||||
| print 'killing ftp process...' | ||||
|
Daniel Valdez
|
r488 | self.status = 0 | ||
| return | ||||
|
Daniel Valdez
|
r517 | self.status = 1 | ||
| return | ||||
|
Daniel Valdez
|
r488 | |||
| def filterByExt(self, ext, localfolder): | ||||
| fnameList = glob.glob1(localfolder,ext) | ||||
| self.filenameList = [os.path.join(localfolder,x) for x in fnameList] | ||||
| if len(self.filenameList) == 0: | ||||
| self.status = 0 | ||||
|
Daniel Valdez
|
r489 | def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1): | ||
|
Daniel Valdez
|
r488 | |||
|
Daniel Valdez
|
r489 | self.counter += 1 | ||
| if self.counter >= period: | ||||
| self.filterByExt(ext, localfolder) | ||||
|
Daniel Valdez
|
r517 | self.ftp_thread(server, username, password, remotefolder) | ||
|
Daniel Valdez
|
r489 | self.counter = 0 | ||
|
Daniel Valdez
|
r488 | |||
|
Daniel Valdez
|
r517 | self.status = 1 | ||