jroutils_ftp.py
1201 lines
| 32.4 KiB
| text/x-python
|
PythonLexer
Daniel Valdez
|
r488 | ''' | ||
| @author: Daniel Suarez | ||||
| ''' | ||||
| import os | ||||
| import glob | ||||
| import ftplib | ||||
Miguel Valdez
|
r573 | |||
| try: | ||||
| import paramiko | ||||
| import scp | ||||
| except: | ||||
Juan C. Espinoza
|
r1284 | pass | ||
|
Miguel Valdez
|
r573 | |||
| import time | ||||
|
Miguel Valdez
|
r633 | import threading | ||
| Thread = threading.Thread | ||||
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r633 | # try: | ||
| # from gevent import sleep | ||||
| # except: | ||||
| from time import sleep | ||||
| r1406 | from schainpy.model.data.jrodata import * | |||
| r1281 | from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator | |||
| r1406 | #@MPDecorator | |||
|
Miguel Valdez
|
r633 | class Remote(Thread): | ||
|
Miguel Valdez
|
r573 | """ | ||
| Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are | ||||
| used to upload or download files remotely. | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | Non-standard Python modules used: | ||
| None | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | Written by: | ||
|
José Chávez
|
r1026 | "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015 | ||
| r1406 | Modified by: | |||
| - | ||||
|
Miguel Valdez
|
r573 | """ | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | server = None | ||
| username = None | ||||
| password = None | ||||
| remotefolder = None | ||||
| r1406 | key_filename=None | |||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | period = 60 | ||
| fileList = [] | ||||
| bussy = False | ||||
|
José Chávez
|
r897 | |||
| r1406 | def __init__(self, server, username, password, remotefolder, period=60,key_filename=None): | |||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r633 | Thread.__init__(self) | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r606 | self.setDaemon(True) | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | self.status = 0 | ||
|
Miguel Valdez
|
r653 | self.__server = server | ||
| self.__username = username | ||||
| self.__password = password | ||||
| self.__remotefolder = remotefolder | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | self.period = period | ||
| r1406 | self.key_filename = key_filename | |||
|
Miguel Valdez
|
r573 | self.fileList = [] | ||
| self.bussy = False | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | self.stopFlag = False | ||
|
José Chávez
|
r897 | |||
George Yong
|
r1167 | print("[Remote Server] Opening server: %s" %self.__server) | ||
| r1406 | if self.open(self.__server, self.__username, self.__password, self.__remotefolder,key_filename=self.key_filename): | |||
|
George Yong
|
r1167 | print("[Remote Server] %s server was opened successfully" %self.__server) | ||
|
José Chávez
|
r897 | |||
| r1406 | #self.close() | |||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r633 | self.mutex = threading.Lock() | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def stop(self): | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | self.stopFlag = True | ||
|
Miguel Valdez
|
r633 | self.join(10) | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def open(self): | ||
| """ | ||||
| Connect to server and create a connection class (FTP or SSH) to remote server. | ||||
| """ | ||||
|
George Yong
|
r1167 | raise NotImplementedError("Implement this method in child class") | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def close(self): | ||
| """ | ||||
| Close connection to server | ||||
| """ | ||||
|
George Yong
|
r1167 | raise NotImplementedError("Implement this method in child class") | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def mkdir(self, remotefolder): | ||
| """ | ||||
| Create a folder remotely | ||||
| """ | ||||
|
George Yong
|
r1167 | raise NotImplementedError("Implement this method in child class") | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def cd(self, remotefolder): | ||
| """ | ||||
| Change working directory in remote server | ||||
| """ | ||||
|
George Yong
|
r1167 | raise NotImplementedError("Implement this method in child class") | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def download(self, filename, localfolder=None): | ||
| """ | ||||
| Download a file from server to local host | ||||
| """ | ||||
|
George Yong
|
r1167 | raise NotImplementedError("Implement this method in child class") | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def sendFile(self, fullfilename): | ||
| """ | ||||
| sendFile method is used to upload a local file to the current directory in remote server | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | Inputs: | ||
| fullfilename - full path name of local file to store in remote directory | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | Returns: | ||
| 0 in error case else 1 | ||||
| """ | ||||
|
George Yong
|
r1167 | raise NotImplementedError("Implement this method in child class") | ||
|
José Chávez
|
r897 | |||
| def upload(self, fullfilename, remotefolder=None): | ||||
|
Miguel Valdez
|
r573 | """ | ||
| upload method is used to upload a local file to remote directory. This method changes | ||||
| working directory before sending a file. | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | Inputs: | ||
| fullfilename - full path name of local file to store in remote directory | ||||
|
José Chávez
|
r897 | |||
| remotefolder - remote directory | ||||
|
Miguel Valdez
|
r573 | Returns: | ||
| 0 in error case else 1 | ||||
| """ | ||||
|
George Yong
|
r1167 | print("[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)) | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if not self.status: | ||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if remotefolder == None: | ||
| remotefolder = self.remotefolder | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if not self.cd(remotefolder): | ||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if not self.sendFile(fullfilename): | ||
|
George Yong
|
r1167 | print("[Remote Server] Error uploading file %s" %fullfilename) | ||
|
Miguel Valdez
|
r573 | return 0 | ||
|
José Chávez
|
r897 | |||
| r1406 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def delete(self, filename): | ||
| """ | ||||
| Remove a file from remote server | ||||
| """ | ||||
| pass | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def updateFileList(self, fileList): | ||
| """ | ||||
| Remove a file from remote server | ||||
| """ | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if fileList == self.fileList: | ||
|
Miguel Valdez
|
r606 | return 0 | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r633 | self.mutex.acquire() | ||
|
José Chávez
|
r1026 | # init = time.time() | ||
| r1279 | # | |||
|
José Chávez
|
r1026 | # while(self.bussy): | ||
| # sleep(0.1) | ||||
| # if time.time() - init > 2*self.period: | ||||
| # return 0 | ||||
| r1279 | ||||
|
Miguel Valdez
|
r573 | self.fileList = fileList | ||
|
Miguel Valdez
|
r633 | self.mutex.release() | ||
|
Miguel Valdez
|
r573 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def run(self): | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r580 | if not self.status: | ||
|
George Yong
|
r1167 | print("Finishing FTP service") | ||
|
Miguel Valdez
|
r580 | return | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if not self.cd(self.remotefolder): | ||
|
George Yong
|
r1167 | raise ValueError("Could not access to the new remote directory: %s" %self.remotefolder) | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | while True: | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r633 | for i in range(self.period): | ||
| if self.stopFlag: | ||||
| break | ||||
| sleep(1) | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r633 | if self.stopFlag: | ||
| break | ||||
| r1279 | ||||
|
José Chávez
|
r1034 | # self.bussy = True | ||
|
Miguel Valdez
|
r633 | self.mutex.acquire() | ||
|
José Chávez
|
r897 | |||
|
George Yong
|
r1167 | print("[Remote Server] Opening %s" %self.__server) | ||
|
Miguel Valdez
|
r653 | if not self.open(self.__server, self.__username, self.__password, self.__remotefolder): | ||
|
Miguel Valdez
|
r661 | self.mutex.release() | ||
|
Miguel Valdez
|
r662 | continue | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | for thisFile in self.fileList: | ||
|
Miguel Valdez
|
r646 | self.upload(thisFile, self.remotefolder) | ||
|
José Chávez
|
r897 | |||
|
George Yong
|
r1167 | print("[Remote Server] Closing %s" %self.__server) | ||
|
Miguel Valdez
|
r653 | self.close() | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r633 | self.mutex.release() | ||
|
José Chávez
|
r1034 | # self.bussy = False | ||
|
José Chávez
|
r897 | |||
|
George Yong
|
r1167 | print("[Remote Server] Thread stopped successfully") | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | class FTPClient(Remote): | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | __ftpClientObj = None | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def __init__(self, server, username, password, remotefolder, period=60): | ||
| """ | ||||
| """ | ||||
| Remote.__init__(self, server, username, password, remotefolder, period) | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def open(self, server, username, password, remotefolder): | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | """ | ||
| This method is used to set FTP parameters and establish a connection to remote server | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | Inputs: | ||
|
José Chávez
|
r897 | server - remote server IP Address | ||
| username - remote server Username | ||||
|
Miguel Valdez
|
r573 | password - remote server password | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | remotefolder - remote server current working directory | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r653 | Return: | ||
| Boolean - Returns 1 if a connection has been established, 0 otherwise | ||||
|
José Chávez
|
r897 | |||
| Affects: | ||||
|
Miguel Valdez
|
r573 | self.status - in case of error or fail connection this parameter is set to 0 else 1 | ||
| """ | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if server == None: | ||
|
George Yong
|
r1167 | raise ValueError("FTP server should be defined") | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if username == None: | ||
|
George Yong
|
r1167 | raise ValueError("FTP username should be defined") | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if password == None: | ||
|
George Yong
|
r1167 | raise ValueError("FTP password should be defined") | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if remotefolder == None: | ||
|
George Yong
|
r1167 | raise ValueError("FTP remote folder should be defined") | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | try: | ||
| ftpClientObj = ftplib.FTP(server) | ||||
|
George Yong
|
r1167 | except ftplib.all_errors as e: | ||
| print("[FTP Server]: FTP server connection fail: %s" %server) | ||||
| print("[FTP Server]:", e) | ||||
|
Miguel Valdez
|
r573 | self.status = 0 | ||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | try: | ||
| ftpClientObj.login(username, password) | ||||
| except ftplib.all_errors: | ||||
|
George Yong
|
r1167 | print("[FTP Server]: FTP username or password are incorrect") | ||
|
Miguel Valdez
|
r573 | self.status = 0 | ||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if remotefolder == None: | ||
| remotefolder = ftpClientObj.pwd() | ||||
| else: | ||||
| try: | ||||
| ftpClientObj.cwd(remotefolder) | ||||
| except ftplib.all_errors: | ||||
|
George Yong
|
r1167 | print("[FTP Server]: FTP remote folder is invalid: %s" %remotefolder) | ||
|
Miguel Valdez
|
r573 | remotefolder = ftpClientObj.pwd() | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | self.server = server | ||
| self.username = username | ||||
| self.password = password | ||||
| self.remotefolder = remotefolder | ||||
| self.__ftpClientObj = ftpClientObj | ||||
| self.status = 1 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def close(self): | ||
| """ | ||||
| Close connection to remote server | ||||
| """ | ||||
| if not self.status: | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | self.__ftpClientObj.close() | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def mkdir(self, remotefolder): | ||
| """ | ||||
| mkdir is used to make a new directory in remote server | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | Input: | ||
| remotefolder - directory name | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | Return: | ||
| 0 in error case else 1 | ||||
| """ | ||||
| if not self.status: | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | try: | ||
| self.__ftpClientObj.mkd(dirname) | ||||
| except ftplib.all_errors: | ||||
|
George Yong
|
r1167 | print("[FTP Server]: Error creating remote folder: %s" %remotefolder) | ||
|
Miguel Valdez
|
r573 | return 0 | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def cd(self, remotefolder): | ||
| """ | ||||
| cd is used to change remote working directory on server | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | Input: | ||
| remotefolder - current working directory | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | Affects: | ||
| self.remotefolder | ||||
|
José Chávez
|
r897 | |||
| Return: | ||||
|
Miguel Valdez
|
r573 | 0 in case of error else 1 | ||
| """ | ||||
| if not self.status: | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if remotefolder == self.remotefolder: | ||
| return 1 | ||||
|
José Chávez
|
r897 | |||
| try: | ||||
| self.__ftpClientObj.cwd(remotefolder) | ||||
|
Miguel Valdez
|
r573 | except ftplib.all_errors: | ||
|
George Yong
|
r1167 | print('[FTP Server]: Error changing to %s' %remotefolder) | ||
| print('[FTP Server]: Trying to create remote folder') | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if not self.mkdir(remotefolder): | ||
|
George Yong
|
r1167 | print('[FTP Server]: Remote folder could not be created') | ||
|
Miguel Valdez
|
r573 | return 0 | ||
|
José Chávez
|
r897 | |||
| try: | ||||
| self.__ftpClientObj.cwd(remotefolder) | ||||
|
Miguel Valdez
|
r573 | except ftplib.all_errors: | ||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | self.remotefolder = remotefolder | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def sendFile(self, fullfilename): | ||
| if not self.status: | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r653 | fp = open(fullfilename, 'rb') | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r606 | filename = os.path.basename(fullfilename) | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | command = "STOR %s" %filename | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | try: | ||
|
Miguel Valdez
|
r653 | self.__ftpClientObj.storbinary(command, fp) | ||
|
George Yong
|
r1167 | except ftplib.all_errors as e: | ||
| print("[FTP Server]:", e) | ||||
|
Miguel Valdez
|
r573 | return 0 | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | try: | ||
| self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename) | ||||
|
George Yong
|
r1167 | except ftplib.all_errors as e: | ||
| print("[FTP Server]:", e) | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r653 | fp.close() | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | return 1 | ||
| class SSHClient(Remote): | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | __sshClientObj = None | ||
| __scpClientObj = None | ||||
|
José Chávez
|
r897 | |||
| r1406 | ||||
| def __init__(self, server, username, password, remotefolder, period=60,key_filename=None): | ||||
|
Miguel Valdez
|
r573 | """ | ||
| """ | ||||
| r1406 | Remote.__init__(self, server, username, password, remotefolder, period, key_filename) | |||
|
José Chávez
|
r897 | |||
| r1406 | def open(self, server, username, password, remotefolder, port=22, key_filename=None): | |||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | """ | ||
|
José Chávez
|
r1026 | This method is used to set SSH parameters and establish a connection to a remote server | ||
| r1279 | ||||
|
José Chávez
|
r1026 | Inputs: | ||
| r1279 | server - remote server IP Address | |||
| username - remote server Username | ||||
|
José Chávez
|
r1026 | password - remote server password | ||
| r1279 | ||||
|
José Chávez
|
r1026 | remotefolder - remote server current working directory | ||
| r1279 | ||||
| r1406 | key_filename - filename of the private key/optional | |||
|
José Chávez
|
r1026 | Return: void | ||
| r1279 | ||||
| Affects: | ||||
|
José Chávez
|
r1026 | self.status - in case of error or fail connection this parameter is set to 0 else 1 | ||
|
Miguel Valdez
|
r573 | |||
| """ | ||||
| r1406 | #import socket | |||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if server == None: | ||
|
George Yong
|
r1167 | raise ValueError("SSH server should be defined") | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if username == None: | ||
|
George Yong
|
r1167 | raise ValueError("SSH username should be defined") | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if password == None: | ||
|
George Yong
|
r1167 | raise ValueError("SSH password should be defined") | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if remotefolder == None: | ||
|
George Yong
|
r1167 | raise ValueError("SSH remote folder should be defined") | ||
|
José Chávez
|
r897 | |||
| r1406 | self.__sshClientObj = paramiko.SSHClient() | |||
|
José Chávez
|
r897 | |||
| r1406 | self.__sshClientObj.load_system_host_keys() | |||
| self.__sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy()) | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r653 | self.status = 0 | ||
| r1406 | ||||
|
Miguel Valdez
|
r573 | try: | ||
| r1406 | if key_filename != None: | |||
| self.__sshClientObj.connect(server, username=username, password=password, port=port, key_filename=key_filename) | ||||
| else: | ||||
| self.__sshClientObj.connect(server, username=username, password=password, port=port) | ||||
|
George Yong
|
r1167 | except paramiko.AuthenticationException as e: | ||
|
José Chávez
|
r1026 | # print "SSH username or password are incorrect: %s" | ||
|
George Yong
|
r1167 | print("[SSH Server]:", e) | ||
|
Miguel Valdez
|
r653 | return 0 | ||
| r1406 | # except SSHException as e: | |||
| # print("[SSH Server]:", e) | ||||
| # return 0 | ||||
| # except socket.error: | ||||
| # self.status = 0 | ||||
| # print("[SSH Server]:", e) | ||||
| # return 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r653 | self.status = 1 | ||
| r1406 | #self.__scpClientObj = scp.SCPClient(self.__sshClientObj.get_transport(), socket_timeout=30) | |||
| self.__scpClientObj = self.__sshClientObj.open_sftp() | ||||
|
Miguel Valdez
|
r573 | if remotefolder == None: | ||
| remotefolder = self.pwd() | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | self.server = server | ||
| self.username = username | ||||
| self.password = password | ||||
| r1406 | # self.__sshClientObj = self.__sshClientObj | |||
| # self.__scpClientObj = self.__scpClientObj | ||||
|
Miguel Valdez
|
r573 | self.status = 1 | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r576 | if not self.cd(remotefolder): | ||
|
George Yong
|
r1167 | raise ValueError("[SSH Server]: Could not access to remote folder: %s" %remotefolder) | ||
|
Miguel Valdez
|
r576 | return 0 | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r576 | self.remotefolder = remotefolder | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def close(self): | ||
| """ | ||||
|
José Chávez
|
r1026 | Close connection to remote server | ||
|
Miguel Valdez
|
r573 | """ | ||
| if not self.status: | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r653 | self.__scpClientObj.close() | ||
| self.__sshClientObj.close() | ||||
|
Miguel Valdez
|
r576 | |||
| def __execute(self, command): | ||||
|
Miguel Valdez
|
r573 | """ | ||
|
José Chávez
|
r1026 | __execute a command on remote server | ||
| r1279 | ||||
|
José Chávez
|
r1026 | Input: | ||
| command - Exmaple 'ls -l' | ||||
| r1279 | ||||
|
José Chávez
|
r1026 | Return: | ||
| 0 in error case else 1 | ||||
|
Miguel Valdez
|
r573 | """ | ||
| if not self.status: | ||||
| return 0 | ||||
|
Miguel Valdez
|
r576 | |||
| stdin, stdout, stderr = self.__sshClientObj.exec_command(command) | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r576 | result = stderr.readlines() | ||
|
Miguel Valdez
|
r573 | if len(result) > 1: | ||
| return 0 | ||||
|
Miguel Valdez
|
r576 | |||
| result = stdout.readlines() | ||||
| if len(result) > 1: | ||||
| return result[0][:-1] | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r576 | def mkdir(self, remotefolder): | ||
| """ | ||||
|
José Chávez
|
r1026 | mkdir is used to make a new directory in remote server | ||
| r1279 | ||||
|
José Chávez
|
r1026 | Input: | ||
| remotefolder - directory name | ||||
| r1279 | ||||
|
José Chávez
|
r1026 | Return: | ||
| 0 in error case else 1 | ||||
|
Miguel Valdez
|
r576 | """ | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r576 | command = 'mkdir %s' %remotefolder | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r576 | return self.__execute(command) | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def pwd(self): | ||
|
Miguel Valdez
|
r576 | command = 'pwd' | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r576 | return self.__execute(command) | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def cd(self, remotefolder): | ||
| """ | ||||
|
José Chávez
|
r1026 | cd is used to change remote working directory on server | ||
| r1279 | ||||
|
José Chávez
|
r1026 | Input: | ||
| remotefolder - current working directory | ||||
| r1279 | ||||
|
José Chávez
|
r1026 | Affects: | ||
| self.remotefolder | ||||
| r1279 | ||||
| Return: | ||||
|
José Chávez
|
r1026 | 0 in case of error else 1 | ||
|
Miguel Valdez
|
r573 | """ | ||
| if not self.status: | ||||
| return 0 | ||||
|
Miguel Valdez
|
r576 | |||
|
Miguel Valdez
|
r573 | if remotefolder == self.remotefolder: | ||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r576 | chk_command = "cd %s; pwd" %remotefolder | ||
| mkdir_command = "mkdir %s" %remotefolder | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r576 | if not self.__execute(chk_command): | ||
| if not self.__execute(mkdir_command): | ||||
| self.remotefolder = None | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | self.remotefolder = remotefolder | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def sendFile(self, fullfilename): | ||
| if not self.status: | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
| r1406 | remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1]) | |||
| print("remotefile",fullfilename, remotefile) | ||||
|
Miguel Valdez
|
r573 | try: | ||
| r1406 | self.__scpClientObj.put(fullfilename,remotefile) | |||
| except paramiko.SSHException as e: | ||||
|
George Yong
|
r1167 | print("[SSH Server]", str(e)) | ||
| r1406 | print(fullfilename," to ",remotefile) | |||
|
Miguel Valdez
|
r573 | return 0 | ||
|
José Chávez
|
r897 | |||
| r1406 | #command = 'chmod 775 %s' %remotefile | |||
| return 1#self.__execute(command) | ||||
| #@MPDecorator | ||||
| class SendToServerProc(ProcessingUnit): | ||||
| sendByTrigger = False | ||||
|
José Chávez
|
r897 | |||
| def __init__(self, **kwargs): | ||||
| r1406 | ProcessingUnit.__init__(self) | |||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | self.isConfig = False | ||
| r1279 | self.clientObj = None | |||
| r1406 | self.dataOut = Parameters() | |||
| self.dataOut.error=False | ||||
| self.dataOut.flagNoData=True | ||||
| r1279 | ||||
| r1406 | def setup(self, server=None, username="", password="", remotefolder="", localfolder="", | |||
| r1420 | ext='.png', period=60, protocol='ftp', sendByTrigger=False, key_filename=None): | |||
| r1292 | self.server = server | |||
| self.username = username | ||||
| self.password = password | ||||
| r1406 | self.remotefolder = remotefolder | |||
|
Miguel Valdez
|
r573 | self.clientObj = None | ||
| self.localfolder = localfolder | ||||
| self.ext = ext | ||||
| r1406 | self.sendByTrigger = sendByTrigger | |||
|
Miguel Valdez
|
r573 | self.period = period | ||
| r1406 | self.key_filename = key_filename | |||
| if self.sendByTrigger: | ||||
| self.period = 1000000000000 #para que no se ejecute por tiempo | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if str.lower(protocol) == 'ftp': | ||
| self.clientObj = FTPClient(server, username, password, remotefolder, period) | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if str.lower(protocol) == 'ssh': | ||
| r1406 | self.clientObj = SSHClient(self.server, self.username, self.password, | |||
| self.remotefolder, period=600000,key_filename=self.key_filename) | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | if not self.clientObj: | ||
|
George Yong
|
r1167 | raise ValueError("%s has been chosen as remote access protocol but it is not valid" %protocol) | ||
|
José Chávez
|
r897 | |||
| r1406 | print("Send to Server setup complete") | |||
|
Miguel Valdez
|
r573 | def findFiles(self): | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r606 | if not type(self.localfolder) == list: | ||
| folderList = [self.localfolder] | ||||
| else: | ||||
| folderList = self.localfolder | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r633 | #Remove duplicate items | ||
| folderList = list(set(folderList)) | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r606 | fullfilenameList = [] | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r606 | for thisFolder in folderList: | ||
|
José Chávez
|
r897 | |||
|
George Yong
|
r1167 | print("[Remote Server]: Searching files on %s" %thisFolder) | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r606 | filenameList = glob.glob1(thisFolder, '*%s' %self.ext) | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r606 | if len(filenameList) < 1: | ||
|
José Chávez
|
r1019 | |||
|
Miguel Valdez
|
r606 | continue | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r606 | for thisFile in filenameList: | ||
| fullfilename = os.path.join(thisFolder, thisFile) | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r633 | if fullfilename in fullfilenameList: | ||
| continue | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r633 | #Only files modified in the last 30 minutes are considered | ||
| if os.path.getmtime(fullfilename) < time.time() - 30*60: | ||||
| continue | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r606 | fullfilenameList.append(fullfilename) | ||
| r1406 | fullfilenameList.sort() | |||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | return fullfilenameList | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def run(self, **kwargs): | ||
| r1406 | ||||
|
Miguel Valdez
|
r573 | if not self.isConfig: | ||
| self.init = time.time() | ||||
| self.setup(**kwargs) | ||||
| self.isConfig = True | ||||
| r1279 | ||||
|
José Chávez
|
r1011 | if not self.clientObj.is_alive(): | ||
|
George Yong
|
r1167 | print("[Remote Server]: Restarting connection ") | ||
| r1406 | self.setup( **kwargs) | |||
| r1279 | ||||
| r1406 | if ((time.time() - self.init) >= self.period and not self.sendByTrigger) or (self.sendByTrigger and not self.dataIn.flagNoData): | |||
|
Miguel Valdez
|
r573 | fullfilenameList = self.findFiles() | ||
| r1406 | if self.sendByTrigger: | |||
| if self.clientObj.upload(fullfilenameList[-1]): #last file to send | ||||
| print("[Remote Server] upload finished successfully") | ||||
| else: | ||||
| for file in fullfilenameList: | ||||
| self.clientObj.upload(file) | ||||
| # if self.clientObj.updateFileList(fullfilenameList): | ||||
| # print("[Remote Server]: Sending the next files ", str(fullfilenameList)) | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | self.init = time.time() | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | def close(self): | ||
|
George Yong
|
r1167 | print("[Remote Server] Stopping thread") | ||
|
Miguel Valdez
|
r573 | self.clientObj.stop() | ||
|
José Chávez
|
r897 | |||
| r1406 | class SendByRSYNCProc(ProcessingUnit): | |||
| sendByTrigger = False | ||||
| def __init__(self, **kwargs): | ||||
| ProcessingUnit.__init__(self) | ||||
| self.isConfig = False | ||||
| self.dataOut = Parameters() | ||||
| self.dataOut.error=False | ||||
| self.dataOut.flagNoData=True | ||||
| def setup(self, server="", username="", remotefolder="", localfolder="",sendByTrigger=True, | ||||
| period=60, key_filename=None, port=22 ,param1="", param2=""): | ||||
| self.server = server | ||||
| self.username = username | ||||
| self.remotefolder = remotefolder | ||||
| self.localfolder = localfolder | ||||
| self.period = period | ||||
| self.key_filename = key_filename | ||||
| if type(param1)==str: | ||||
| self.param1 = list(param1.split(",")) | ||||
| else: | ||||
| self.param1 = param1 | ||||
| if type(param2)==str: | ||||
| self.param2 = list(param2.split(",")) | ||||
| else: | ||||
| self.param2 = param2 | ||||
| self.port = port | ||||
| self.sendByTrigger = sendByTrigger | ||||
| if self.sendByTrigger: | ||||
| self.period = 1000000000000 #para que no se ejecute por tiempo | ||||
| self.command ="rsync " | ||||
| def syncFolders(self): | ||||
| self.command ="rsync " | ||||
| for p1 in self.param1: | ||||
| self.command += " -"+str(p1) | ||||
| for p2 in self.param2: | ||||
| self.command += " --"+str(p2) | ||||
| if self.key_filename != None: | ||||
| self.command += """ "ssh -i {} -p {}" """.format(self.key_filename, self.port) | ||||
| self.command += " {} ".format(self.localfolder) | ||||
| self.command += " {}@{}:{}".format(self.username,self.server,self.remotefolder) | ||||
| print("CMD: ",self.command) | ||||
| #os.system(self.command) | ||||
| return | ||||
| def run(self, **kwargs): | ||||
| if not self.isConfig: | ||||
| self.init = time.time() | ||||
| self.setup(**kwargs) | ||||
| self.isConfig = True | ||||
| if self.sendByTrigger and not self.dataIn.flagNoData: | ||||
| self.syncFolders() | ||||
| else: | ||||
| if (time.time() - self.init) >= self.period: | ||||
| self.syncFolders() | ||||
| self.init = time.time() | ||||
| return | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r517 | class FTP(object): | ||
|
Daniel Valdez
|
r488 | """ | ||
| Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Non-standard Python modules used: None | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010 | ||
| r1422 | ||||
| Modified: | ||||
| Joab Apaza Feb. 2022 | ||||
|
Daniel Valdez
|
r488 | """ | ||
|
José Chávez
|
r897 | |||
| def __init__(self,server = None, username=None, password=None, remotefolder=None): | ||||
|
Daniel Valdez
|
r488 | """ | ||
| This method is used to setting parameters for FTP and establishing connection to remote server | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Inputs: | ||
|
José Chávez
|
r897 | server - remote server IP Address | ||
| username - remote server Username | ||||
|
Daniel Valdez
|
r488 | password - remote server password | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | remotefolder - remote server current working directory | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Return: void | ||
|
José Chávez
|
r897 | |||
| Affects: | ||||
|
Daniel Valdez
|
r488 | self.status - in Error Case or Connection Failed this parameter is set to 1 else 0 | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | self.folderList - sub-folder list of remote folder | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | self.fileList - file list of remote folder | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | """ | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | if ((server == None) and (username==None) and (password==None) and (remotefolder==None)): | ||
| server, username, password, remotefolder = self.parmsByDefault() | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | self.server = server | ||
| self.username = username | ||||
| self.password = password | ||||
| self.remotefolder = remotefolder | ||||
| self.file = None | ||||
| self.ftp = None | ||||
| self.status = 0 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | try: | ||
| self.ftp = ftplib.FTP(self.server) | ||||
| self.ftp.login(self.username,self.password) | ||||
| r1279 | self.ftp.cwd(self.remotefolder) | |||
|
José Chávez
|
r1034 | # print 'Connect to FTP Server: Successfully' | ||
| r1279 | ||||
|
Daniel Valdez
|
r488 | except ftplib.all_errors: | ||
|
George Yong
|
r1167 | print('Error FTP Service') | ||
|
Daniel Valdez
|
r488 | self.status = 1 | ||
| return | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | self.dirList = [] | ||
| try: | ||||
| self.dirList = self.ftp.nlst() | ||||
|
José Chávez
|
r897 | |||
|
George Yong
|
r1167 | except ftplib.error_perm as resp: | ||
|
Daniel Valdez
|
r488 | if str(resp) == "550 No files found": | ||
|
George Yong
|
r1167 | print("no files in this directory") | ||
|
Daniel Valdez
|
r488 | self.status = 1 | ||
| return | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | except ftplib.all_errors: | ||
|
George Yong
|
r1167 | print('Error Displaying Dir-Files') | ||
|
Daniel Valdez
|
r488 | self.status = 1 | ||
| return | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | self.fileList = [] | ||
| self.folderList = [] | ||||
| #only for test | ||||
| for f in self.dirList: | ||||
| name, ext = os.path.splitext(f) | ||||
| if ext != '': | ||||
| self.fileList.append(f) | ||||
|
José Chávez
|
r1026 | # print 'filename: %s - size: %d'%(f,self.ftp.size(f)) | ||
|
Daniel Valdez
|
r488 | |||
| def parmsByDefault(self): | ||||
| server = 'jro-app.igp.gob.pe' | ||||
| username = 'wmaster' | ||||
| password = 'mst2010vhf' | ||||
| remotefolder = '/home/wmaster/graficos' | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | return server, username, password, remotefolder | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | def mkd(self,dirname): | ||
| """ | ||||
| mkd is used to make directory in remote server | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Input: | ||
| dirname - directory name | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Return: | ||
| 1 in error case else 0 | ||||
| """ | ||||
|
José Chávez
|
r897 | try: | ||
|
Daniel Valdez
|
r488 | self.ftp.mkd(dirname) | ||
| except: | ||||
|
George Yong
|
r1167 | print('Error creating remote folder:%s'%dirname) | ||
| r1422 | return False | |||
|
José Chávez
|
r897 | |||
| r1422 | return True | |||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | def delete(self,filename): | ||
| """ | ||||
| delete is used to delete file in current working directory of remote server | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Input: | ||
| filename - filename to delete in remote folder | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Return: | ||
| 1 in error case else 0 | ||||
| """ | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | try: | ||
| self.ftp.delete(filename) | ||||
| except: | ||||
|
George Yong
|
r1167 | print('Error deleting remote file:%s'%filename) | ||
| r1422 | return False | |||
|
José Chávez
|
r897 | |||
| r1422 | return True | |||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | def download(self,filename,localfolder): | ||
| """ | ||||
| download is used to downloading file from remote folder into local folder | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Inputs: | ||
| filename - filename to donwload | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | localfolder - directory local to store filename | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Returns: | ||
| self.status - 1 in error case else 0 | ||||
| """ | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | self.status = 0 | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | if not(filename in self.fileList): | ||
|
George Yong
|
r1167 | print('filename:%s not exists'%filename) | ||
|
Daniel Valdez
|
r488 | self.status = 1 | ||
| return self.status | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | newfilename = os.path.join(localfolder,filename) | ||
|
José Chávez
|
r897 | |||
| self.file = open(newfilename, 'wb') | ||||
|
Daniel Valdez
|
r488 | try: | ||
|
George Yong
|
r1167 | print('Download: ' + filename) | ||
|
Daniel Valdez
|
r488 | self.ftp.retrbinary('RETR ' + filename, self.__handleDownload) | ||
|
George Yong
|
r1167 | print('Download Complete') | ||
|
Daniel Valdez
|
r488 | except ftplib.all_errors: | ||
|
George Yong
|
r1167 | print('Error Downloading ' + filename) | ||
|
Daniel Valdez
|
r488 | self.status = 1 | ||
| return self.status | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | self.file.close() | ||
|
José Chávez
|
r897 | |||
| return self.status | ||||
|
Daniel Valdez
|
r488 | |||
| def __handleDownload(self,block): | ||||
| """ | ||||
| __handleDownload is used to handle writing file | ||||
| """ | ||||
| self.file.write(block) | ||||
|
José Chávez
|
r897 | |||
| r1422 | def upload(self,filename,remotefolder=None, mkdir=False): | |||
|
Daniel Valdez
|
r488 | """ | ||
| r1422 | upload is used to uploading local file to remote directory, and change the permission of the remote file | |||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Inputs: | ||
| filename - full path name of local file to store in remote directory | ||||
|
José Chávez
|
r897 | |||
| remotefolder - remote directory | ||||
| r1422 | mkdir - if the remote folder doesn't exist, it will created | |||
|
Daniel Valdez
|
r488 | Returns: | ||
| self.status - 1 in error case else 0 | ||||
| """ | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | if remotefolder == None: | ||
| remotefolder = self.remotefolder | ||||
|
José Chávez
|
r897 | |||
| r1422 | if mkdir: | |||
| if self.if_dir_exist(remotefolder): | ||||
| pass | ||||
| else: | ||||
| self.mkdir_r(remotefolder) | ||||
|
Daniel Valdez
|
r488 | self.status = 0 | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | try: | ||
|
José Chávez
|
r897 | self.ftp.cwd(remotefolder) | ||
|
Daniel Valdez
|
r488 | self.file = open(filename, 'rb') | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | (head, tail) = os.path.split(filename) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | command = "STOR " + tail | ||
|
José Chávez
|
r897 | |||
|
George Yong
|
r1167 | print('Uploading: ' + tail) | ||
|
José Chávez
|
r897 | self.ftp.storbinary(command, self.file) | ||
| r1422 | print(self.cmd('SITE CHMOD 755 {}'.format(tail))) | |||
|
George Yong
|
r1167 | print('Upload Completed') | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | except ftplib.all_errors: | ||
|
George Yong
|
r1167 | print('Error Uploading ' + tail) | ||
|
Daniel Valdez
|
r488 | self.status = 1 | ||
| return self.status | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | self.file.close() | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | #back to initial directory in __init__() | ||
|
José Chávez
|
r897 | self.ftp.cwd(self.remotefolder) | ||
| return self.status | ||||
| r1422 | def ch_dir(self,remotefolder): | |||
|
Daniel Valdez
|
r488 | """ | ||
| r1422 | ch_dir is used to change working directory of remote server and get folder and file list | |||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Input: | ||
| remotefolder - current working directory | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Affects: | ||
| self.fileList - file list of working directory | ||||
|
José Chávez
|
r897 | |||
| Return: | ||||
|
Daniel Valdez
|
r488 | infoList - list with filenames and size of file in bytes | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | self.folderList - folder list | ||
| """ | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | self.remotefolder = remotefolder | ||
|
George Yong
|
r1167 | print('Change to ' + self.remotefolder) | ||
|
José Chávez
|
r897 | try: | ||
| self.ftp.cwd(remotefolder) | ||||
|
Daniel Valdez
|
r488 | except ftplib.all_errors: | ||
|
George Yong
|
r1167 | print('Error Change to ' + self.remotefolder) | ||
|
Daniel Valdez
|
r488 | infoList = None | ||
| self.folderList = None | ||||
| return infoList,self.folderList | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | self.dirList = [] | ||
| try: | ||||
| self.dirList = self.ftp.nlst() | ||||
|
José Chávez
|
r897 | |||
|
George Yong
|
r1167 | except ftplib.error_perm as resp: | ||
|
José Chávez
|
r897 | if str(resp) == "550 No files found": | ||
|
George Yong
|
r1167 | print("no files in this directory") | ||
|
Daniel Valdez
|
r488 | infoList = None | ||
| self.folderList = None | ||||
| return infoList,self.folderList | ||||
| except ftplib.all_errors: | ||||
|
George Yong
|
r1167 | print('Error Displaying Dir-Files') | ||
|
Daniel Valdez
|
r488 | infoList = None | ||
| self.folderList = None | ||||
|
José Chávez
|
r897 | return infoList,self.folderList | ||
| infoList = [] | ||||
|
Daniel Valdez
|
r488 | self.fileList = [] | ||
| self.folderList = [] | ||||
| for f in self.dirList: | ||||
| name,ext = os.path.splitext(f) | ||||
|
José Chávez
|
r897 | if ext != '': | ||
|
Daniel Valdez
|
r488 | self.fileList.append(f) | ||
| value = (f,self.ftp.size(f)) | ||||
|
José Chávez
|
r897 | infoList.append(value) | ||
| if ext == '': | ||||
|
Daniel Valdez
|
r488 | self.folderList.append(f) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | return infoList,self.folderList | ||
| def close(self): | ||||
| """ | ||||
| close is used to close and end FTP connection | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Inputs: None | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | Return: void | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | """ | ||
| self.ftp.close() | ||||
| r1422 | ||||
| def get_sub_dirs(self, path): | ||||
| """ | ||||
| used internal | ||||
| Inputs: | ||||
| path - path to split in sub folders | ||||
| Returns: | ||||
| sub_dirs - list of sub folders | ||||
| """ | ||||
| sub_dirs = path.split("/") | ||||
| if sub_dirs[0]=="/": | ||||
| sub_dirs.pop(0) | ||||
| if sub_dirs[-1]=="/": | ||||
| sub_dirs.pop(-1) | ||||
| return sub_dirs | ||||
| def if_dir_exist(self,path): | ||||
| """ | ||||
| check if a the path folder exists in the ftp server | ||||
| Inputs: | ||||
| path - path to check | ||||
| Returns: | ||||
| status - True if exists and False if it doesn't | ||||
| """ | ||||
| sub_dirs = self.get_sub_dirs(path) | ||||
| main = self.ftp.pwd() | ||||
| #print(main) | ||||
| for subdir in sub_dirs: | ||||
| folders = self.ftp.nlst(main) | ||||
| #print(folders) | ||||
| if (os.path.join(main,subdir) in folders): | ||||
| main = os.path.join(main,subdir) | ||||
| #print(main) | ||||
| continue | ||||
| else: | ||||
| return False | ||||
| return True | ||||
| def cmd(self,command): | ||||
| """ | ||||
| excecute a command in the FTP server | ||||
| """ | ||||
| return self.ftp.sendcmd(command) | ||||
| def mkdir_r(self,path): | ||||
| """ | ||||
| create a remote folder and create sub folders if it is necessary | ||||
| Inputs: | ||||
| path - path to create | ||||
| Returns: | ||||
| status - True if succesfull else False | ||||
| """ | ||||
| sub_dirs = self.get_sub_dirs(path) | ||||
| main = self.ftp.pwd() | ||||
| st = False | ||||
| #print(main) | ||||
| for subdir in sub_dirs: | ||||
| folders = self.ftp.nlst(main) | ||||
| #print(folders) | ||||
| folder = (os.path.join(main,subdir)) | ||||
| if (folder in folders): | ||||
| main = folder | ||||
| #print("new_main",main) | ||||
| continue | ||||
| else: | ||||
| print("creating...",folder) | ||||
| st = self.mkd(folder) | ||||
| print(self.cmd('SITE CHMOD 755 {}'.format(folder))) | ||||
| main = folder | ||||
| return st | ||||
| r1292 | @MPDecorator | |||
|
Daniel Valdez
|
r488 | class SendByFTP(Operation): | ||
|
José Chávez
|
r897 | |||
| def __init__(self, **kwargs): | ||||
| Operation.__init__(self, **kwargs) | ||||
|
Daniel Valdez
|
r488 | self.status = 1 | ||
|
Daniel Valdez
|
r489 | self.counter = 0 | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | def error_print(self, ValueError): | ||
|
José Chávez
|
r897 | |||
|
George Yong
|
r1167 | print(ValueError, 'Error FTP') | ||
| print("don't worry the program is running...") | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r517 | def worker_ftp(self, server, username, password, remotefolder, filenameList): | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r573 | self.ftpClientObj = FTP(server, username, password, remotefolder) | ||
|
Daniel Valdez
|
r517 | for filename in filenameList: | ||
|
Miguel Valdez
|
r573 | self.ftpClientObj.upload(filename) | ||
| self.ftpClientObj.close() | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r517 | def ftp_thread(self, server, username, password, remotefolder): | ||
|
Daniel Valdez
|
r488 | if not(self.status): | ||
| return | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r633 | import multiprocessing | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r517 | p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,)) | ||
| p.start() | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r517 | p.join(3) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r517 | if p.is_alive(): | ||
| p.terminate() | ||||
| p.join() | ||||
|
George Yong
|
r1167 | print('killing ftp process...') | ||
|
Daniel Valdez
|
r488 | self.status = 0 | ||
| return | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r517 | self.status = 1 | ||
| return | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r488 | def filterByExt(self, ext, localfolder): | ||
| fnameList = glob.glob1(localfolder,ext) | ||||
| self.filenameList = [os.path.join(localfolder,x) for x in fnameList] | ||||
| if len(self.filenameList) == 0: | ||||
| self.status = 0 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r489 | def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1): | ||
|
José Chávez
|
r897 | |||
| self.counter += 1 | ||||
|
Daniel Valdez
|
r489 | if self.counter >= period: | ||
| self.filterByExt(ext, localfolder) | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r517 | self.ftp_thread(server, username, password, remotefolder) | ||
|
Daniel Valdez
|
r489 | self.counter = 0 | ||
|
José Chávez
|
r897 | |||
| r1279 | self.status = 1 | |||
