schain Commit - r1209:53188d049ed7 · Jicamarca Repository

Fix SendToFTP operation for v3

Juan C. Espinoza -

r1209:53188d049ed7

parent child

Context file:

r1209:53188d049ed7

Collapse all files

schainpy/model/graphics/jroplot_data.py +1 0

                     self.titles = ['{} {}'.format(
                         self.data.parameters[x], title) for x in self.channels]
             class ScopePlot(Plot):
                 '''

schainpy/model/utils/jroutils_publish.py +67 -50

             from threading import Thread
             from multiprocessing import Process
-            from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
+            from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit, MPDecorator
             from schainpy.model.data.jrodata import JROData
             from schainpy.utils import log
                 __attrs__ = ['host', 'port', 'delay', 'verbose']
-                def __init__(self, **kwargs):
-                    """Inicio."""
-                    Operation.__init__(self, **kwargs)
-                    self.isConfig = False
                 def setup(self, server='zmq.pipe', delay=0, verbose=True, **kwargs):
                     self.counter = 0
                     self.delay = kwargs.get('delay', 0)
                     self.cnt = 0
                     self.verbose = verbose
-                    setup = []
                     context = zmq.Context()
                     self.zmq_socket = context.socket(zmq.PUSH)
                     server = kwargs.get('server', 'zmq.pipe')
                                              self.dataOut.datatime.ctime(),),
                             'Receiving')
+            @MPDecorator
-            class SendToFTP(Operation, Process):
+            class SendToFTP(Operation):
                 '''
                 Operation to send data over FTP.
+                patternX = 'local, remote, ext, period, exp_code, sub_exp_code'
                 '''
-                __attrs__ = ['server', 'username', 'password', 'patterns', 'timeout']
+                __attrs__ = ['server', 'username', 'password', 'timeout', 'patternX']
-                def __init__(self, **kwargs):
+                def __init__(self):
                     '''
-                    patterns = [(local1, remote1, ext, delay, exp_code, sub_exp_code), ...]
                     '''
-                    Operation.__init__(self, **kwargs)
+                    Operation.__init__(self)
-                    Process.__init__(self)
-                    self.server = kwargs.get('server')
-                    self.username = kwargs.get('username')
-                    self.password = kwargs.get('password')
-                    self.patterns = kwargs.get('patterns')
-                    self.timeout = kwargs.get('timeout', 30)
-                    self.times = [time.time() for p in self.patterns]
-                    self.latest = ['' for p in self.patterns]
-                    self.mp = False
                     self.ftp = None
+                    self.ready = False
-                def setup(self):
+                def setup(self, server, username, password, timeout, **kwargs):
+                    '''
+                    '''
+                    self.server = server
+                    self.username = username
+                    self.password = password
+                    self.timeout = timeout
+                    self.patterns = []
+                    self.times = []
+                    self.latest = []
+                    for arg, value in kwargs.items():
+                        if 'pattern' in arg:
+                            self.patterns.append(value)
+                            self.times.append(time.time())
+                            self.latest.append('')
+                def connect(self):
+                    '''
+                    '''
                     log.log('Connecting to ftp://{}'.format(self.server), self.name)
                     try:
                         if self.ftp is not None:
                             self.ftp.close()
                         self.ftp = None
-                        self.isConfig = False
+                        self.ready = False
                         return
                     try:
                         if self.ftp is not None:
                             self.ftp.close()
                         self.ftp = None
-                        self.isConfig = False
+                        self.ready = False
                         return
                     log.success('Connection success', self.name)
-                    self.isConfig = True
+                    self.ready = True
                     return
                 def check(self):
                         if self.ftp is not None:
                             self.ftp.close()
                         self.ftp = None
-                        self.setup()
+                        self.connect()
                 def find_files(self, path, ext):
                 def getftpname(self, filename, exp_code, sub_exp_code):
                     thisDatetime = datetime.datetime.strptime(filename.split('_')[1], '%Y%m%d')
-                    YEAR_STR = '%4.4d'%thisDatetime.timetuple().tm_year
+                    YEAR_STR = '%4.4d' % thisDatetime.timetuple().tm_year
-                    DOY_STR = '%3.3d'%thisDatetime.timetuple().tm_yday
+                    DOY_STR = '%3.3d' % thisDatetime.timetuple().tm_yday
-                    exp_code = '%3.3d'%exp_code
+                    exp_code = '%3.3d' % exp_code
-                    sub_exp_code = '%2.2d'%sub_exp_code
+                    sub_exp_code = '%2.2d' % sub_exp_code
-                    plot_code = '%2.2d'% get_plot_code(filename)
+                    plot_code = '%2.2d' % get_plot_code(filename)
                     name = YEAR_STR + DOY_STR + '00' + exp_code + sub_exp_code + plot_code + '00.png'
                     return name
                 def upload(self, src, dst):
-                    log.log('Uploading {} '.format(src), self.name, nl=False)
+                    log.log('Uploading {} -> {} '.format(
+                        src.split('/')[-1], dst.split('/')[-1]),
+                        self.name,
+                        nl=False
+                        )
                     fp = open(src, 'rb')
                     command = 'STOR {}'.format(dst)
                 def send_files(self):
                     for x, pattern in enumerate(self.patterns):
-                        local, remote, ext, delay, exp_code, sub_exp_code = pattern
+                        local, remote, ext, period, exp_code, sub_exp_code = pattern
-                        if time.time()-self.times[x] >= delay:
+                        if time.time()-self.times[x] >= int(period):
                             srcname = self.find_files(local, ext)
                             src = os.path.join(local, srcname)
                             if os.path.getmtime(src) < time.time() - 30*60:
+                                log.warning('Skipping old file {}'.format(srcname))
                                 continue
                             if srcname is None or srcname == self.latest[x]:
+                                log.warning('File alreday uploaded {}'.format(srcname))
                                 continue
                             if 'png' in ext:
-                                dstname = self.getftpname(srcname, exp_code, sub_exp_code)
+                                dstname = self.getftpname(srcname, int(exp_code), int(sub_exp_code))
                             else:
                                 dstname = srcname
                                 self.times[x] = time.time()
                                 self.latest[x] = srcname
                             else:
-                                self.isConfig = False
+                                self.ready = False
                                 break
-                def run(self):
+                def run(self, dataOut, server, username, password, timeout=10, **kwargs):
-                    while True:
+                    if not self.isConfig:
-                        if not self.isConfig:
+                        self.setup(
-                            self.setup()
+                            server=server,
-                        if self.ftp is not None:
+                            username=username,
-                            self.check()
+                            password=password,
-                            self.send_files()
+                            timeout=timeout,
-                        time.sleep(10)
+                            **kwargs
+                            )
+                        self.isConfig = True
+                    if not self.ready:
+                        self.connect()
+                    if self.ftp is not None:
+                        self.check()
+                        self.send_files()
-                def close():
+                def close(self):
                     if self.ftp is not None:
                         self.ftp.close()
-                    self.terminate()
  No newline at end of file

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages