jroutils_ftp.py
1007 lines
| 26.6 KiB
| text/x-python
|
PythonLexer
|
r488 | ''' | |
@author: Daniel Suarez | |||
''' | |||
import os | |||
import glob | |||
import ftplib | |||
|
r573 | ||
try: | |||
import paramiko | |||
import scp | |||
except: | |||
|
r1167 | print("You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server") | |
|
r573 | ||
import time | |||
|
r633 | import threading | |
Thread = threading.Thread | |||
|
r897 | ||
|
r633 | # try: | |
# from gevent import sleep | |||
# except: | |||
from time import sleep | |||
|
r897 | ||
|
r568 | from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation | |
|
r488 | ||
|
r633 | class Remote(Thread): | |
|
r573 | """ | |
Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are | |||
used to upload or download files remotely. | |||
|
r897 | ||
|
r573 | Non-standard Python modules used: | |
None | |||
|
r897 | ||
|
r573 | Written by: | |
|
r1026 | "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015 | |
|
r573 | """ | |
|
r897 | ||
|
r573 | server = None | |
username = None | |||
password = None | |||
remotefolder = None | |||
|
r897 | ||
|
r573 | period = 60 | |
fileList = [] | |||
bussy = False | |||
|
r897 | ||
|
r573 | def __init__(self, server, username, password, remotefolder, period=60): | |
|
r897 | ||
|
r633 | Thread.__init__(self) | |
|
r897 | ||
|
r606 | self.setDaemon(True) | |
|
r897 | ||
|
r573 | self.status = 0 | |
|
r653 | self.__server = server | |
self.__username = username | |||
self.__password = password | |||
self.__remotefolder = remotefolder | |||
|
r897 | ||
|
r573 | self.period = period | |
|
r897 | ||
|
r573 | self.fileList = [] | |
self.bussy = False | |||
|
r897 | ||
|
r573 | self.stopFlag = False | |
|
r897 | ||
|
r1167 | print("[Remote Server] Opening server: %s" %self.__server) | |
|
r653 | if self.open(self.__server, self.__username, self.__password, self.__remotefolder): | |
|
r1167 | print("[Remote Server] %s server was opened successfully" %self.__server) | |
|
r897 | ||
|
r653 | self.close() | |
|
r897 | ||
|
r633 | self.mutex = threading.Lock() | |
|
r897 | ||
|
r573 | def stop(self): | |
|
r897 | ||
|
r573 | self.stopFlag = True | |
|
r633 | self.join(10) | |
|
r897 | ||
|
r573 | def open(self): | |
""" | |||
Connect to server and create a connection class (FTP or SSH) to remote server. | |||
""" | |||
|
r1167 | raise NotImplementedError("Implement this method in child class") | |
|
r897 | ||
|
r573 | def close(self): | |
""" | |||
Close connection to server | |||
""" | |||
|
r1167 | raise NotImplementedError("Implement this method in child class") | |
|
r897 | ||
|
r573 | def mkdir(self, remotefolder): | |
""" | |||
Create a folder remotely | |||
""" | |||
|
r1167 | raise NotImplementedError("Implement this method in child class") | |
|
r897 | ||
|
r573 | def cd(self, remotefolder): | |
""" | |||
Change working directory in remote server | |||
""" | |||
|
r1167 | raise NotImplementedError("Implement this method in child class") | |
|
r897 | ||
|
r573 | def download(self, filename, localfolder=None): | |
""" | |||
Download a file from server to local host | |||
""" | |||
|
r1167 | raise NotImplementedError("Implement this method in child class") | |
|
r897 | ||
|
r573 | def sendFile(self, fullfilename): | |
""" | |||
sendFile method is used to upload a local file to the current directory in remote server | |||
|
r897 | ||
|
r573 | Inputs: | |
fullfilename - full path name of local file to store in remote directory | |||
|
r897 | ||
|
r573 | Returns: | |
0 in error case else 1 | |||
""" | |||
|
r1167 | raise NotImplementedError("Implement this method in child class") | |
|
r897 | ||
def upload(self, fullfilename, remotefolder=None): | |||
|
r573 | """ | |
upload method is used to upload a local file to remote directory. This method changes | |||
working directory before sending a file. | |||
|
r897 | ||
|
r573 | Inputs: | |
fullfilename - full path name of local file to store in remote directory | |||
|
r897 | ||
remotefolder - remote directory | |||
|
r573 | Returns: | |
0 in error case else 1 | |||
""" | |||
|
r1167 | print("[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)) | |
|
r897 | ||
|
r573 | if not self.status: | |
return 0 | |||
|
r897 | ||
|
r573 | if remotefolder == None: | |
remotefolder = self.remotefolder | |||
|
r897 | ||
|
r573 | if not self.cd(remotefolder): | |
return 0 | |||
|
r897 | ||
|
r573 | if not self.sendFile(fullfilename): | |
|
r1167 | print("[Remote Server] Error uploading file %s" %fullfilename) | |
|
r573 | return 0 | |
|
r897 | ||
|
r1167 | print("[Remote Server] upload finished successfully") | |
|
r897 | ||
|
r573 | return 1 | |
|
r897 | ||
|
r573 | def delete(self, filename): | |
""" | |||
Remove a file from remote server | |||
""" | |||
pass | |||
|
r897 | ||
|
r573 | def updateFileList(self, fileList): | |
""" | |||
Remove a file from remote server | |||
""" | |||
|
r897 | ||
|
r573 | if fileList == self.fileList: | |
|
r606 | return 0 | |
|
r897 | ||
|
r633 | self.mutex.acquire() | |
|
r1026 | # init = time.time() | |
# | |||
# while(self.bussy): | |||
# sleep(0.1) | |||
# if time.time() - init > 2*self.period: | |||
# return 0 | |||
|
r573 | self.fileList = fileList | |
|
r633 | self.mutex.release() | |
|
r573 | return 1 | |
|
r897 | ||
|
r573 | def run(self): | |
|
r897 | ||
|
r580 | if not self.status: | |
|
r1167 | print("Finishing FTP service") | |
|
r580 | return | |
|
r897 | ||
|
r573 | if not self.cd(self.remotefolder): | |
|
r1167 | raise ValueError("Could not access to the new remote directory: %s" %self.remotefolder) | |
|
r897 | ||
|
r573 | while True: | |
|
r897 | ||
|
r633 | for i in range(self.period): | |
if self.stopFlag: | |||
break | |||
sleep(1) | |||
|
r897 | ||
|
r633 | if self.stopFlag: | |
break | |||
|
r1034 | # self.bussy = True | |
|
r633 | self.mutex.acquire() | |
|
r897 | ||
|
r1167 | print("[Remote Server] Opening %s" %self.__server) | |
|
r653 | if not self.open(self.__server, self.__username, self.__password, self.__remotefolder): | |
|
r661 | self.mutex.release() | |
|
r662 | continue | |
|
r897 | ||
|
r573 | for thisFile in self.fileList: | |
|
r646 | self.upload(thisFile, self.remotefolder) | |
|
r897 | ||
|
r1167 | print("[Remote Server] Closing %s" %self.__server) | |
|
r653 | self.close() | |
|
r897 | ||
|
r633 | self.mutex.release() | |
|
r1034 | # self.bussy = False | |
|
r897 | ||
|
r1167 | print("[Remote Server] Thread stopped successfully") | |
|
r897 | ||
|
r573 | class FTPClient(Remote): | |
|
r897 | ||
|
r573 | __ftpClientObj = None | |
|
r897 | ||
|
r573 | def __init__(self, server, username, password, remotefolder, period=60): | |
""" | |||
""" | |||
Remote.__init__(self, server, username, password, remotefolder, period) | |||
|
r897 | ||
|
r573 | def open(self, server, username, password, remotefolder): | |
|
r897 | ||
|
r573 | """ | |
This method is used to set FTP parameters and establish a connection to remote server | |||
|
r897 | ||
|
r573 | Inputs: | |
|
r897 | server - remote server IP Address | |
username - remote server Username | |||
|
r573 | password - remote server password | |
|
r897 | ||
|
r573 | remotefolder - remote server current working directory | |
|
r897 | ||
|
r653 | Return: | |
Boolean - Returns 1 if a connection has been established, 0 otherwise | |||
|
r897 | ||
Affects: | |||
|
r573 | self.status - in case of error or fail connection this parameter is set to 0 else 1 | |
""" | |||
|
r897 | ||
|
r573 | if server == None: | |
|
r1167 | raise ValueError("FTP server should be defined") | |
|
r897 | ||
|
r573 | if username == None: | |
|
r1167 | raise ValueError("FTP username should be defined") | |
|
r897 | ||
|
r573 | if password == None: | |
|
r1167 | raise ValueError("FTP password should be defined") | |
|
r897 | ||
|
r573 | if remotefolder == None: | |
|
r1167 | raise ValueError("FTP remote folder should be defined") | |
|
r897 | ||
|
r573 | try: | |
ftpClientObj = ftplib.FTP(server) | |||
|
r1167 | except ftplib.all_errors as e: | |
print("[FTP Server]: FTP server connection fail: %s" %server) | |||
print("[FTP Server]:", e) | |||
|
r573 | self.status = 0 | |
return 0 | |||
|
r897 | ||
|
r573 | try: | |
ftpClientObj.login(username, password) | |||
except ftplib.all_errors: | |||
|
r1167 | print("[FTP Server]: FTP username or password are incorrect") | |
|
r573 | self.status = 0 | |
return 0 | |||
|
r897 | ||
|
r573 | if remotefolder == None: | |
remotefolder = ftpClientObj.pwd() | |||
else: | |||
try: | |||
ftpClientObj.cwd(remotefolder) | |||
except ftplib.all_errors: | |||
|
r1167 | print("[FTP Server]: FTP remote folder is invalid: %s" %remotefolder) | |
|
r573 | remotefolder = ftpClientObj.pwd() | |
|
r897 | ||
|
r573 | self.server = server | |
self.username = username | |||
self.password = password | |||
self.remotefolder = remotefolder | |||
self.__ftpClientObj = ftpClientObj | |||
self.status = 1 | |||
|
r897 | ||
|
r573 | return 1 | |
|
r897 | ||
|
r573 | def close(self): | |
""" | |||
Close connection to remote server | |||
""" | |||
if not self.status: | |||
return 0 | |||
|
r897 | ||
|
r573 | self.__ftpClientObj.close() | |
|
r897 | ||
|
r573 | def mkdir(self, remotefolder): | |
""" | |||
mkdir is used to make a new directory in remote server | |||
|
r897 | ||
|
r573 | Input: | |
remotefolder - directory name | |||
|
r897 | ||
|
r573 | Return: | |
0 in error case else 1 | |||
""" | |||
if not self.status: | |||
return 0 | |||
|
r897 | ||
|
r573 | try: | |
self.__ftpClientObj.mkd(dirname) | |||
except ftplib.all_errors: | |||
|
r1167 | print("[FTP Server]: Error creating remote folder: %s" %remotefolder) | |
|
r573 | return 0 | |
|
r897 | ||
|
r573 | return 1 | |
|
r897 | ||
|
r573 | def cd(self, remotefolder): | |
""" | |||
cd is used to change remote working directory on server | |||
|
r897 | ||
|
r573 | Input: | |
remotefolder - current working directory | |||
|
r897 | ||
|
r573 | Affects: | |
self.remotefolder | |||
|
r897 | ||
Return: | |||
|
r573 | 0 in case of error else 1 | |
""" | |||
if not self.status: | |||
return 0 | |||
|
r897 | ||
|
r573 | if remotefolder == self.remotefolder: | |
return 1 | |||
|
r897 | ||
try: | |||
self.__ftpClientObj.cwd(remotefolder) | |||
|
r573 | except ftplib.all_errors: | |
|
r1167 | print('[FTP Server]: Error changing to %s' %remotefolder) | |
print('[FTP Server]: Trying to create remote folder') | |||
|
r897 | ||
|
r573 | if not self.mkdir(remotefolder): | |
|
r1167 | print('[FTP Server]: Remote folder could not be created') | |
|
r573 | return 0 | |
|
r897 | ||
try: | |||
self.__ftpClientObj.cwd(remotefolder) | |||
|
r573 | except ftplib.all_errors: | |
return 0 | |||
|
r897 | ||
|
r573 | self.remotefolder = remotefolder | |
|
r897 | ||
|
r573 | return 1 | |
|
r897 | ||
|
r573 | def sendFile(self, fullfilename): | |
if not self.status: | |||
return 0 | |||
|
r897 | ||
|
r653 | fp = open(fullfilename, 'rb') | |
|
r897 | ||
|
r606 | filename = os.path.basename(fullfilename) | |
|
r897 | ||
|
r573 | command = "STOR %s" %filename | |
|
r897 | ||
|
r573 | try: | |
|
r653 | self.__ftpClientObj.storbinary(command, fp) | |
|
r1167 | except ftplib.all_errors as e: | |
print("[FTP Server]:", e) | |||
|
r573 | return 0 | |
|
r897 | ||
|
r573 | try: | |
self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename) | |||
|
r1167 | except ftplib.all_errors as e: | |
print("[FTP Server]:", e) | |||
|
r897 | ||
|
r653 | fp.close() | |
|
r897 | ||
|
r573 | return 1 | |
class SSHClient(Remote): | |||
|
r897 | ||
|
r573 | __sshClientObj = None | |
__scpClientObj = None | |||
|
r897 | ||
|
r573 | def __init__(self, server, username, password, remotefolder, period=60): | |
""" | |||
""" | |||
Remote.__init__(self, server, username, password, remotefolder, period) | |||
|
r897 | ||
|
r573 | def open(self, server, username, password, remotefolder, port=22): | |
|
r897 | ||
|
r573 | """ | |
|
r1026 | This method is used to set SSH parameters and establish a connection to a remote server | |
|
r573 | ||
|
r1026 | Inputs: | |
server - remote server IP Address | |||
username - remote server Username | |||
password - remote server password | |||
remotefolder - remote server current working directory | |||
|
r573 | ||
|
r1026 | Return: void | |
|
r573 | ||
|
r1026 | Affects: | |
self.status - in case of error or fail connection this parameter is set to 0 else 1 | |||
|
r573 | ||
""" | |||
|
r653 | import socket | |
|
r897 | ||
|
r573 | if server == None: | |
|
r1167 | raise ValueError("SSH server should be defined") | |
|
r897 | ||
|
r573 | if username == None: | |
|
r1167 | raise ValueError("SSH username should be defined") | |
|
r897 | ||
|
r573 | if password == None: | |
|
r1167 | raise ValueError("SSH password should be defined") | |
|
r897 | ||
|
r573 | if remotefolder == None: | |
|
r1167 | raise ValueError("SSH remote folder should be defined") | |
|
r897 | ||
|
r653 | sshClientObj = paramiko.SSHClient() | |
|
r897 | ||
|
r573 | sshClientObj.load_system_host_keys() | |
sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy()) | |||
|
r897 | ||
|
r653 | self.status = 0 | |
|
r573 | try: | |
|
r897 | sshClientObj.connect(server, username=username, password=password, port=port) | |
|
r1167 | except paramiko.AuthenticationException as e: | |
|
r1026 | # print "SSH username or password are incorrect: %s" | |
|
r1167 | print("[SSH Server]:", e) | |
|
r653 | return 0 | |
|
r1167 | except SSHException as e: | |
print("[SSH Server]:", e) | |||
|
r653 | return 0 | |
except socket.error: | |||
|
r573 | self.status = 0 | |
|
r1167 | print("[SSH Server]:", e) | |
|
r573 | return 0 | |
|
r897 | ||
|
r653 | self.status = 1 | |
|
r573 | scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30) | |
|
r897 | ||
|
r573 | if remotefolder == None: | |
remotefolder = self.pwd() | |||
|
r897 | ||
|
r573 | self.server = server | |
self.username = username | |||
self.password = password | |||
self.__sshClientObj = sshClientObj | |||
self.__scpClientObj = scpClientObj | |||
self.status = 1 | |||
|
r897 | ||
|
r576 | if not self.cd(remotefolder): | |
|
r1167 | raise ValueError("[SSH Server]: Could not access to remote folder: %s" %remotefolder) | |
|
r576 | return 0 | |
|
r897 | ||
|
r576 | self.remotefolder = remotefolder | |
|
r897 | ||
|
r573 | return 1 | |
|
r897 | ||
|
r573 | def close(self): | |
""" | |||
|
r1026 | Close connection to remote server | |
|
r573 | """ | |
if not self.status: | |||
return 0 | |||
|
r897 | ||
|
r653 | self.__scpClientObj.close() | |
self.__sshClientObj.close() | |||
|
r576 | ||
def __execute(self, command): | |||
|
r573 | """ | |
|
r1026 | __execute a command on remote server | |
Input: | |||
command - Exmaple 'ls -l' | |||
Return: | |||
0 in error case else 1 | |||
|
r573 | """ | |
if not self.status: | |||
return 0 | |||
|
r576 | ||
stdin, stdout, stderr = self.__sshClientObj.exec_command(command) | |||
|
r897 | ||
|
r576 | result = stderr.readlines() | |
|
r573 | if len(result) > 1: | |
return 0 | |||
|
r576 | ||
result = stdout.readlines() | |||
if len(result) > 1: | |||
return result[0][:-1] | |||
|
r897 | ||
|
r573 | return 1 | |
|
r897 | ||
|
r576 | def mkdir(self, remotefolder): | |
""" | |||
|
r1026 | mkdir is used to make a new directory in remote server | |
Input: | |||
remotefolder - directory name | |||
Return: | |||
0 in error case else 1 | |||
|
r576 | """ | |
|
r897 | ||
|
r576 | command = 'mkdir %s' %remotefolder | |
|
r897 | ||
|
r576 | return self.__execute(command) | |
|
r897 | ||
|
r573 | def pwd(self): | |
|
r576 | command = 'pwd' | |
|
r897 | ||
|
r576 | return self.__execute(command) | |
|
r897 | ||
|
r573 | def cd(self, remotefolder): | |
""" | |||
|
r1026 | cd is used to change remote working directory on server | |
|
r573 | ||
|
r1026 | Input: | |
remotefolder - current working directory | |||
Affects: | |||
self.remotefolder | |||
Return: | |||
0 in case of error else 1 | |||
|
r573 | """ | |
if not self.status: | |||
return 0 | |||
|
r576 | ||
|
r573 | if remotefolder == self.remotefolder: | |
return 1 | |||
|
r897 | ||
|
r576 | chk_command = "cd %s; pwd" %remotefolder | |
mkdir_command = "mkdir %s" %remotefolder | |||
|
r897 | ||
|
r576 | if not self.__execute(chk_command): | |
if not self.__execute(mkdir_command): | |||
self.remotefolder = None | |||
return 0 | |||
|
r897 | ||
|
r573 | self.remotefolder = remotefolder | |
|
r897 | ||
|
r573 | return 1 | |
|
r897 | ||
|
r573 | def sendFile(self, fullfilename): | |
if not self.status: | |||
return 0 | |||
|
r897 | ||
|
r573 | try: | |
self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder) | |||
|
r1167 | except scp.ScpError as e: | |
print("[SSH Server]", str(e)) | |||
|
r573 | return 0 | |
|
r897 | ||
|
r576 | remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1]) | |
command = 'chmod 775 %s' %remotefile | |||
|
r897 | ||
|
r576 | return self.__execute(command) | |
|
r897 | ||
|
r573 | class SendToServer(ProcessingUnit): | |
|
r897 | ||
def __init__(self, **kwargs): | |||
ProcessingUnit.__init__(self, **kwargs) | |||
|
r573 | self.isConfig = False | |
|
r1026 | self.clientObj = None | |
|
r573 | ||
|
r581 | def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs): | |
|
r897 | ||
|
r573 | self.clientObj = None | |
self.localfolder = localfolder | |||
self.ext = ext | |||
self.period = period | |||
|
r897 | ||
|
r573 | if str.lower(protocol) == 'ftp': | |
self.clientObj = FTPClient(server, username, password, remotefolder, period) | |||
|
r897 | ||
|
r573 | if str.lower(protocol) == 'ssh': | |
self.clientObj = SSHClient(server, username, password, remotefolder, period) | |||
|
r897 | ||
|
r573 | if not self.clientObj: | |
|
r1167 | raise ValueError("%s has been chosen as remote access protocol but it is not valid" %protocol) | |
|
r897 | ||
|
r573 | self.clientObj.start() | |
|
r897 | ||
|
r573 | def findFiles(self): | |
|
r897 | ||
|
r606 | if not type(self.localfolder) == list: | |
folderList = [self.localfolder] | |||
else: | |||
folderList = self.localfolder | |||
|
r897 | ||
|
r633 | #Remove duplicate items | |
folderList = list(set(folderList)) | |||
|
r897 | ||
|
r606 | fullfilenameList = [] | |
|
r897 | ||
|
r606 | for thisFolder in folderList: | |
|
r897 | ||
|
r1167 | print("[Remote Server]: Searching files on %s" %thisFolder) | |
|
r897 | ||
|
r606 | filenameList = glob.glob1(thisFolder, '*%s' %self.ext) | |
|
r897 | ||
|
r606 | if len(filenameList) < 1: | |
|
r1019 | ||
|
r606 | continue | |
|
r897 | ||
|
r606 | for thisFile in filenameList: | |
fullfilename = os.path.join(thisFolder, thisFile) | |||
|
r897 | ||
|
r633 | if fullfilename in fullfilenameList: | |
continue | |||
|
r897 | ||
|
r633 | #Only files modified in the last 30 minutes are considered | |
if os.path.getmtime(fullfilename) < time.time() - 30*60: | |||
continue | |||
|
r897 | ||
|
r606 | fullfilenameList.append(fullfilename) | |
|
r897 | ||
|
r573 | return fullfilenameList | |
|
r897 | ||
|
r573 | def run(self, **kwargs): | |
if not self.isConfig: | |||
self.init = time.time() | |||
self.setup(**kwargs) | |||
self.isConfig = True | |||
|
r1011 | if not self.clientObj.is_alive(): | |
|
r1167 | print("[Remote Server]: Restarting connection ") | |
|
r1011 | self.setup(**kwargs) | |
|
r573 | if time.time() - self.init >= self.period: | |
fullfilenameList = self.findFiles() | |||
|
r897 | ||
|
r606 | if self.clientObj.updateFileList(fullfilenameList): | |
|
r1167 | print("[Remote Server]: Sending the next files ", str(fullfilenameList)) | |
|
r573 | self.init = time.time() | |
|
r897 | ||
|
r573 | def close(self): | |
|
r1167 | print("[Remote Server] Stopping thread") | |
|
r573 | self.clientObj.stop() | |
|
r897 | ||
|
r517 | class FTP(object): | |
|
r488 | """ | |
Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module | |||
|
r897 | ||
|
r488 | Non-standard Python modules used: None | |
|
r897 | ||
|
r488 | Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010 | |
""" | |||
|
r897 | ||
def __init__(self,server = None, username=None, password=None, remotefolder=None): | |||
|
r488 | """ | |
This method is used to setting parameters for FTP and establishing connection to remote server | |||
|
r897 | ||
|
r488 | Inputs: | |
|
r897 | server - remote server IP Address | |
username - remote server Username | |||
|
r488 | password - remote server password | |
|
r897 | ||
|
r488 | remotefolder - remote server current working directory | |
|
r897 | ||
|
r488 | Return: void | |
|
r897 | ||
Affects: | |||
|
r488 | self.status - in Error Case or Connection Failed this parameter is set to 1 else 0 | |
|
r897 | ||
|
r488 | self.folderList - sub-folder list of remote folder | |
|
r897 | ||
|
r488 | self.fileList - file list of remote folder | |
|
r897 | ||
|
r488 | """ | |
|
r897 | ||
|
r488 | if ((server == None) and (username==None) and (password==None) and (remotefolder==None)): | |
server, username, password, remotefolder = self.parmsByDefault() | |||
|
r897 | ||
|
r488 | self.server = server | |
self.username = username | |||
self.password = password | |||
self.remotefolder = remotefolder | |||
self.file = None | |||
self.ftp = None | |||
self.status = 0 | |||
|
r897 | ||
|
r488 | try: | |
self.ftp = ftplib.FTP(self.server) | |||
self.ftp.login(self.username,self.password) | |||
self.ftp.cwd(self.remotefolder) | |||
|
r1034 | # print 'Connect to FTP Server: Successfully' | |
|
r488 | ||
except ftplib.all_errors: | |||
|
r1167 | print('Error FTP Service') | |
|
r488 | self.status = 1 | |
return | |||
|
r897 | ||
|
r488 | self.dirList = [] | |
try: | |||
self.dirList = self.ftp.nlst() | |||
|
r897 | ||
|
r1167 | except ftplib.error_perm as resp: | |
|
r488 | if str(resp) == "550 No files found": | |
|
r1167 | print("no files in this directory") | |
|
r488 | self.status = 1 | |
return | |||
|
r897 | ||
|
r488 | except ftplib.all_errors: | |
|
r1167 | print('Error Displaying Dir-Files') | |
|
r488 | self.status = 1 | |
return | |||
|
r897 | ||
|
r488 | self.fileList = [] | |
self.folderList = [] | |||
#only for test | |||
for f in self.dirList: | |||
name, ext = os.path.splitext(f) | |||
if ext != '': | |||
self.fileList.append(f) | |||
|
r1026 | # print 'filename: %s - size: %d'%(f,self.ftp.size(f)) | |
|
r488 | ||
def parmsByDefault(self): | |||
server = 'jro-app.igp.gob.pe' | |||
username = 'wmaster' | |||
password = 'mst2010vhf' | |||
remotefolder = '/home/wmaster/graficos' | |||
|
r897 | ||
|
r488 | return server, username, password, remotefolder | |
|
r897 | ||
|
r488 | def mkd(self,dirname): | |
""" | |||
mkd is used to make directory in remote server | |||
|
r897 | ||
|
r488 | Input: | |
dirname - directory name | |||
|
r897 | ||
|
r488 | Return: | |
1 in error case else 0 | |||
""" | |||
|
r897 | try: | |
|
r488 | self.ftp.mkd(dirname) | |
except: | |||
|
r1167 | print('Error creating remote folder:%s'%dirname) | |
|
r488 | return 1 | |
|
r897 | ||
|
r488 | return 0 | |
|
r897 | ||
|
r488 | def delete(self,filename): | |
""" | |||
delete is used to delete file in current working directory of remote server | |||
|
r897 | ||
|
r488 | Input: | |
filename - filename to delete in remote folder | |||
|
r897 | ||
|
r488 | Return: | |
1 in error case else 0 | |||
""" | |||
|
r897 | ||
|
r488 | try: | |
self.ftp.delete(filename) | |||
except: | |||
|
r1167 | print('Error deleting remote file:%s'%filename) | |
|
r488 | return 1 | |
|
r897 | ||
|
r488 | return 0 | |
|
r897 | ||
|
r488 | def download(self,filename,localfolder): | |
""" | |||
download is used to downloading file from remote folder into local folder | |||
|
r897 | ||
|
r488 | Inputs: | |
filename - filename to donwload | |||
|
r897 | ||
|
r488 | localfolder - directory local to store filename | |
|
r897 | ||
|
r488 | Returns: | |
self.status - 1 in error case else 0 | |||
""" | |||
|
r897 | ||
|
r488 | self.status = 0 | |
|
r897 | ||
|
r488 | if not(filename in self.fileList): | |
|
r1167 | print('filename:%s not exists'%filename) | |
|
r488 | self.status = 1 | |
return self.status | |||
|
r897 | ||
|
r488 | newfilename = os.path.join(localfolder,filename) | |
|
r897 | ||
self.file = open(newfilename, 'wb') | |||
|
r488 | try: | |
|
r1167 | print('Download: ' + filename) | |
|
r488 | self.ftp.retrbinary('RETR ' + filename, self.__handleDownload) | |
|
r1167 | print('Download Complete') | |
|
r488 | except ftplib.all_errors: | |
|
r1167 | print('Error Downloading ' + filename) | |
|
r488 | self.status = 1 | |
return self.status | |||
|
r897 | ||
|
r488 | self.file.close() | |
|
r897 | ||
return self.status | |||
|
r488 | ||
def __handleDownload(self,block): | |||
""" | |||
__handleDownload is used to handle writing file | |||
""" | |||
self.file.write(block) | |||
|
r897 | ||
|
r488 | def upload(self,filename,remotefolder=None): | |
""" | |||
upload is used to uploading local file to remote directory | |||
|
r897 | ||
|
r488 | Inputs: | |
filename - full path name of local file to store in remote directory | |||
|
r897 | ||
remotefolder - remote directory | |||
|
r488 | Returns: | |
self.status - 1 in error case else 0 | |||
""" | |||
|
r897 | ||
|
r488 | if remotefolder == None: | |
remotefolder = self.remotefolder | |||
|
r897 | ||
|
r488 | self.status = 0 | |
|
r897 | ||
|
r488 | try: | |
|
r897 | self.ftp.cwd(remotefolder) | |
|
r488 | self.file = open(filename, 'rb') | |
|
r897 | ||
|
r488 | (head, tail) = os.path.split(filename) | |
|
r897 | ||
|
r488 | command = "STOR " + tail | |
|
r897 | ||
|
r1167 | print('Uploading: ' + tail) | |
|
r897 | self.ftp.storbinary(command, self.file) | |
|
r1167 | print('Upload Completed') | |
|
r897 | ||
|
r488 | except ftplib.all_errors: | |
|
r1167 | print('Error Uploading ' + tail) | |
|
r488 | self.status = 1 | |
return self.status | |||
|
r897 | ||
|
r488 | self.file.close() | |
|
r897 | ||
|
r488 | #back to initial directory in __init__() | |
|
r897 | self.ftp.cwd(self.remotefolder) | |
return self.status | |||
|
r488 | def dir(self,remotefolder): | |
""" | |||
dir is used to change working directory of remote server and get folder and file list | |||
|
r897 | ||
|
r488 | Input: | |
remotefolder - current working directory | |||
|
r897 | ||
|
r488 | Affects: | |
self.fileList - file list of working directory | |||
|
r897 | ||
Return: | |||
|
r488 | infoList - list with filenames and size of file in bytes | |
|
r897 | ||
|
r488 | self.folderList - folder list | |
""" | |||
|
r897 | ||
|
r488 | self.remotefolder = remotefolder | |
|
r1167 | print('Change to ' + self.remotefolder) | |
|
r897 | try: | |
self.ftp.cwd(remotefolder) | |||
|
r488 | except ftplib.all_errors: | |
|
r1167 | print('Error Change to ' + self.remotefolder) | |
|
r488 | infoList = None | |
self.folderList = None | |||
return infoList,self.folderList | |||
|
r897 | ||
|
r488 | self.dirList = [] | |
try: | |||
self.dirList = self.ftp.nlst() | |||
|
r897 | ||
|
r1167 | except ftplib.error_perm as resp: | |
|
r897 | if str(resp) == "550 No files found": | |
|
r1167 | print("no files in this directory") | |
|
r488 | infoList = None | |
self.folderList = None | |||
return infoList,self.folderList | |||
except ftplib.all_errors: | |||
|
r1167 | print('Error Displaying Dir-Files') | |
|
r488 | infoList = None | |
self.folderList = None | |||
|
r897 | return infoList,self.folderList | |
infoList = [] | |||
|
r488 | self.fileList = [] | |
self.folderList = [] | |||
for f in self.dirList: | |||
name,ext = os.path.splitext(f) | |||
|
r897 | if ext != '': | |
|
r488 | self.fileList.append(f) | |
value = (f,self.ftp.size(f)) | |||
|
r897 | infoList.append(value) | |
if ext == '': | |||
|
r488 | self.folderList.append(f) | |
|
r897 | ||
|
r488 | return infoList,self.folderList | |
|
r897 | ||
|
r488 | def close(self): | |
""" | |||
close is used to close and end FTP connection | |||
|
r897 | ||
|
r488 | Inputs: None | |
|
r897 | ||
|
r488 | Return: void | |
|
r897 | ||
|
r488 | """ | |
self.ftp.close() | |||
class SendByFTP(Operation): | |||
|
r897 | ||
def __init__(self, **kwargs): | |||
Operation.__init__(self, **kwargs) | |||
|
r488 | self.status = 1 | |
|
r489 | self.counter = 0 | |
|
r897 | ||
|
r488 | def error_print(self, ValueError): | |
|
r897 | ||
|
r1167 | print(ValueError, 'Error FTP') | |
print("don't worry the program is running...") | |||
|
r897 | ||
|
r517 | def worker_ftp(self, server, username, password, remotefolder, filenameList): | |
|
r897 | ||
|
r573 | self.ftpClientObj = FTP(server, username, password, remotefolder) | |
|
r517 | for filename in filenameList: | |
|
r573 | self.ftpClientObj.upload(filename) | |
self.ftpClientObj.close() | |||
|
r897 | ||
|
r517 | def ftp_thread(self, server, username, password, remotefolder): | |
|
r488 | if not(self.status): | |
return | |||
|
r897 | ||
|
r633 | import multiprocessing | |
|
r897 | ||
|
r517 | p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,)) | |
p.start() | |||
|
r897 | ||
|
r517 | p.join(3) | |
|
r897 | ||
|
r517 | if p.is_alive(): | |
p.terminate() | |||
p.join() | |||
|
r1167 | print('killing ftp process...') | |
|
r488 | self.status = 0 | |
return | |||
|
r897 | ||
|
r517 | self.status = 1 | |
return | |||
|
r897 | ||
|
r488 | def filterByExt(self, ext, localfolder): | |
fnameList = glob.glob1(localfolder,ext) | |||
self.filenameList = [os.path.join(localfolder,x) for x in fnameList] | |||
if len(self.filenameList) == 0: | |||
self.status = 0 | |||
|
r897 | ||
|
r489 | def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1): | |
|
r897 | ||
self.counter += 1 | |||
|
r489 | if self.counter >= period: | |
self.filterByExt(ext, localfolder) | |||
|
r897 | ||
|
r517 | self.ftp_thread(server, username, password, remotefolder) | |
|
r489 | self.counter = 0 | |
|
r897 | ||
|
r1167 | self.status = 1 |