schain Commit - r1422:161fd8f163b6 · Jicamarca Repository

FTP modificado, para usarse libremente (no schain) y subir rti de AMISR a jro-app, ventaja de usar esta clase antigua solo se sube un archivo una vez finalizado el procesamiento offline.

joabAM -

r1422:161fd8f163b6

parent child

Context file:

r1422:161fd8f163b6

Collapse all files

schainpy/controller.py +1 0

             # Copyright (c) 2012-2020 Jicamarca Radio Observatory
             # All rights reserved.
             #
             # Distributed under the terms of the BSD 3-clause license.
             """API to create signal chain projects
             The API is provide through class: Project
             """
             import re
             import sys
             import ast
             import datetime
             import traceback
             import time
             import multiprocessing
             from multiprocessing import Process, Queue
             from threading import Thread
             from xml.etree.ElementTree import ElementTree, Element, SubElement
             from schainpy.admin import Alarm, SchainWarning
             from schainpy.model import *
             from schainpy.utils import log
             if 'darwin' in sys.platform and sys.version_info[0] == 3 and sys.version_info[1] > 7:
                 multiprocessing.set_start_method('fork')
             class ConfBase():
                 def __init__(self):
                     self.id = '0'
                     self.name = None
                     self.priority = None
                     self.parameters = {}
                     self.object = None
                     self.operations = []
                 def getId(self):
                     return self.id
                 def getNewId(self):
                     return int(self.id) * 10 + len(self.operations) + 1
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     n = 1
                     for conf in self.operations:
                         conf_id = str(int(new_id) * 10 + n)
                         conf.updateId(conf_id)
                         n += 1
                 def getKwargs(self):
                     params = {}
                     for key, value in self.parameters.items():
                         if value not in (None, '', ' '):
                             params[key] = value
                     return params
                 def update(self, **kwargs):
                     for key, value in kwargs.items():
                         self.addParameter(name=key, value=value)
                 def addParameter(self, name, value, format=None):
                     '''
                     '''
                     if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
                         self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
                     elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
                         self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
                     else:
                         try:
                             self.parameters[name] = ast.literal_eval(value)
                         except:
                             if isinstance(value, str) and ',' in value:
                                 self.parameters[name] = value.split(',')
                             else:
                                 self.parameters[name] = value
                 def getParameters(self):
                     params = {}
                     for key, value in self.parameters.items():
                         s = type(value).__name__
                         if s == 'date':
                             params[key] = value.strftime('%Y/%m/%d')
                         elif s == 'time':
                             params[key] = value.strftime('%H:%M:%S')
                         else:
                             params[key] = str(value)
                     return params
                 def makeXml(self, element):
                     xml = SubElement(element, self.ELEMENTNAME)
                     for label in self.xml_labels:
                         xml.set(label, str(getattr(self, label)))
                     for key, value in self.getParameters().items():
                         xml_param = SubElement(xml, 'Parameter')
                         xml_param.set('name', key)
                         xml_param.set('value', value)
                     for conf in self.operations:
                         conf.makeXml(xml)
                 def __str__(self):
                     if self.ELEMENTNAME == 'Operation':
                         s = '  {}[id={}]\n'.format(self.name, self.id)
                     else:
                         s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
                     for key, value in self.parameters.items():
                         if self.ELEMENTNAME == 'Operation':
                             s += '    {}: {}\n'.format(key, value)
                         else:
                             s += '  {}: {}\n'.format(key, value)
                     for conf in self.operations:
                         s += str(conf)
                     return s
             class OperationConf(ConfBase):
                 ELEMENTNAME = 'Operation'
                 xml_labels = ['id', 'name']
                 def setup(self, id, name, priority, project_id, err_queue):
                     self.id = str(id)
                     self.project_id = project_id
                     self.name = name
                     self.type = 'other'
                     self.err_queue = err_queue
                 def readXml(self, element, project_id, err_queue):
                     self.id = element.get('id')
                     self.name = element.get('name')
                     self.type = 'other'
                     self.project_id = str(project_id)
                     self.err_queue = err_queue
                     for elm in element.iter('Parameter'):
                         self.addParameter(elm.get('name'), elm.get('value'))
                 def createObject(self):
                     className = eval(self.name)
                     if 'Plot' in self.name or 'Writer' in self.name or 'Send' in self.name or 'print' in self.name:
                         kwargs = self.getKwargs()
                         opObj = className(self.name, self.id, self.project_id, self.err_queue, **kwargs)
                         opObj.start()
                         self.type = 'external'
                     else:
                         opObj = className()
                     self.object = opObj
                     return opObj
             class ProcUnitConf(ConfBase):
                 ELEMENTNAME = 'ProcUnit'
                 xml_labels = ['id', 'inputId', 'name']
                 def setup(self, project_id, id, name, datatype, inputId, err_queue):
                     '''
                     '''
                     if datatype == None and name == None:
                         raise ValueError('datatype or name should be defined')
                     if name == None:
                         if 'Proc' in datatype:
                             name = datatype
                         else:
                             name = '%sProc' % (datatype)
                     if datatype == None:
                         datatype = name.replace('Proc', '')
                     self.id = str(id)
                     self.project_id = project_id
                     self.name = name
                     self.datatype = datatype
                     self.inputId = inputId
                     self.err_queue = err_queue
                     self.operations = []
                     self.parameters = {}
                 def removeOperation(self, id):
                     i = [1 if x.id==id else 0 for x in self.operations]
                     self.operations.pop(i.index(1))
                 def getOperation(self, id):
                     for conf in self.operations:
                         if conf.id == id:
                             return conf
                 def addOperation(self, name, optype='self'):
                     '''
                     '''
                     id = self.getNewId()
                     conf = OperationConf()
                     conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
                     self.operations.append(conf)
                     return conf
                 def readXml(self, element, project_id, err_queue):
                     self.id = element.get('id')
                     self.name = element.get('name')
                     self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
                     self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
                     self.project_id = str(project_id)
                     self.err_queue = err_queue
                     self.operations = []
                     self.parameters = {}
                     for elm in element:
                         if elm.tag == 'Parameter':
                             self.addParameter(elm.get('name'), elm.get('value'))
                         elif elm.tag == 'Operation':
                             conf = OperationConf()
                             conf.readXml(elm, project_id, err_queue)
                             self.operations.append(conf)
                 def createObjects(self):
                     '''
                     Instancia de unidades de procesamiento.
                     '''
                     className = eval(self.name)
                     kwargs = self.getKwargs()
                     procUnitObj = className()
                     procUnitObj.name = self.name
                     log.success('creating process... id: {}, inputId: {}'.format(self.id,self.inputId), self.name)
                     for conf in self.operations:
                         opObj = conf.createObject()
                         log.success('adding operation: {}, type:{}'.format(conf.name,conf.type), self.name)
                         procUnitObj.addOperation(conf, opObj)
                     self.object = procUnitObj
                 def run(self):
                     '''
                     '''
                     return self.object.call(**self.getKwargs())
             class ReadUnitConf(ProcUnitConf):
                 ELEMENTNAME = 'ReadUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.operations = []
                     self.parameters = {}
                 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
                           startTime='', endTime='', server=None, **kwargs):
                     if datatype == None and name == None:
                         raise ValueError('datatype or name should be defined')
                     if name == None:
                         if 'Reader' in datatype:
                             name = datatype
                             datatype = name.replace('Reader','')
                         else:
                             name = '{}Reader'.format(datatype)
                     if datatype == None:
                         if 'Reader' in name:
                             datatype = name.replace('Reader','')
                         else:
                             datatype = name
                             name = '{}Reader'.format(name)
                     self.id = id
                     self.project_id = project_id
                     self.name = name
                     self.datatype = datatype
                     self.err_queue = err_queue
                     self.addParameter(name='path', value=path)
                     self.addParameter(name='startDate', value=startDate)
                     self.addParameter(name='endDate', value=endDate)
                     self.addParameter(name='startTime', value=startTime)
                     self.addParameter(name='endTime', value=endTime)
                     for key, value in kwargs.items():
                         self.addParameter(name=key, value=value)
             class Project(Process):
                 """API to create signal chain projects"""
                 ELEMENTNAME = 'Project'
                 def __init__(self, name=''):
                     Process.__init__(self)
                     self.id = '1'
                     if name:
                         self.name = '{} ({})'.format(Process.__name__, name)
                     self.filename = None
                     self.description = None
                     self.email = None
                     self.alarm = []
                     self.configurations = {}
                     # self.err_queue = Queue()
                     self.err_queue = None
                     self.started = False
                 def getNewId(self):
                     idList = list(self.configurations.keys())
                     id = int(self.id) * 10
                     while True:
                         id += 1
                         if str(id) in idList:
                             continue
                         break
                     return str(id)
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     keyList = list(self.configurations.keys())
                     keyList.sort()
                     n = 1
                     new_confs = {}
                     for procKey in keyList:
                         conf = self.configurations[procKey]
                         idProcUnit = str(int(self.id) * 10 + n)
                         conf.updateId(idProcUnit)
                         new_confs[idProcUnit] = conf
                         n += 1
                     self.configurations = new_confs
                 def setup(self, id=1, name='', description='', email=None, alarm=[]):
                     self.id = str(id)
                     self.description = description
                     self.email = email
                     self.alarm = alarm
                     if name:
                         self.name = '{} ({})'.format(Process.__name__, name)
                 def update(self, **kwargs):
                     for key, value in kwargs.items():
                         setattr(self, key, value)
                 def clone(self):
                     p = Project()
                     p.id = self.id
                     p.name = self.name
                     p.description = self.description
                     p.configurations = self.configurations.copy()
                     return p
                 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
                     '''
                     '''
                     if id is None:
                         idReadUnit = self.getNewId()
                     else:
                         idReadUnit = str(id)
                     conf = ReadUnitConf()
                     conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
                     self.configurations[conf.id] = conf
                     return conf
                 def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
                     '''
                     '''
                     if id is None:
                         idProcUnit = self.getNewId()
                     else:
                         idProcUnit = id
                     conf = ProcUnitConf()
                     conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
                     self.configurations[conf.id] = conf
                     return conf
                 def removeProcUnit(self, id):
                     if id in self.configurations:
                         self.configurations.pop(id)
                 def getReadUnit(self):
                     for obj in list(self.configurations.values()):
                         if obj.ELEMENTNAME == 'ReadUnit':
                             return obj
                     return None
                 def getProcUnit(self, id):
                     return self.configurations[id]
                 def getUnits(self):
                     keys = list(self.configurations)
                     keys.sort()
                     for key in keys:
                         yield self.configurations[key]
                 def updateUnit(self, id, **kwargs):
                     conf = self.configurations[id].update(**kwargs)
                 def makeXml(self):
                     xml = Element('Project')
                     xml.set('id', str(self.id))
                     xml.set('name', self.name)
                     xml.set('description', self.description)
                     for conf in self.configurations.values():
                         conf.makeXml(xml)
                     self.xml = xml
                 def writeXml(self, filename=None):
                     if filename == None:
                         if self.filename:
                             filename = self.filename
                         else:
                             filename = 'schain.xml'
                     if not filename:
                         print('filename has not been defined. Use setFilename(filename) for do it.')
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.access(os.path.dirname(abs_file), os.W_OK):
                         print('No write permission on %s' % os.path.dirname(abs_file))
                         return 0
                     if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
                         print('File %s already exists and it could not be overwriten' % abs_file)
                         return 0
                     self.makeXml()
                     ElementTree(self.xml).write(abs_file, method='xml')
                     self.filename = abs_file
                     return 1
                 def readXml(self, filename):
                     abs_file = os.path.abspath(filename)
                     self.configurations = {}
                     try:
                         self.xml = ElementTree().parse(abs_file)
                     except:
                         log.error('Error reading %s, verify file format' % filename)
                         return 0
                     self.id = self.xml.get('id')
                     self.name = self.xml.get('name')
                     self.description = self.xml.get('description')
                     for element in self.xml:
                         if element.tag == 'ReadUnit':
                             conf = ReadUnitConf()
                             conf.readXml(element, self.id, self.err_queue)
                             self.configurations[conf.id] = conf
                         elif element.tag == 'ProcUnit':
                             conf = ProcUnitConf()
                             input_proc = self.configurations[element.get('inputId')]
                             conf.readXml(element, self.id, self.err_queue)
                             self.configurations[conf.id] = conf
                     self.filename = abs_file
                     return 1
                 def __str__(self):
                     text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
                         self.id,
                         self.name,
                         self.description,
                         )
                     for conf in self.configurations.values():
                         text += '{}'.format(conf)
                     return text
                 def createObjects(self):
                     keys = list(self.configurations.keys())
                     keys.sort()
                     for key in keys:
                         conf = self.configurations[key]
                         conf.createObjects()
                         if conf.inputId is not None:
                             conf.object.setInput(self.configurations[conf.inputId].object)
                 def monitor(self):
                     t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
                     t.start()
                 def _monitor(self, queue, ctx):
                     import socket
                     procs = 0
                     err_msg = ''
                     while True:
                         msg = queue.get()
                         if '#_start_#' in msg:
                             procs += 1
                         elif '#_end_#' in msg:
                             procs -=1
                         else:
                             err_msg = msg
                         if procs == 0 or 'Traceback' in err_msg:
                             break
                         time.sleep(0.1)
                     if '|' in err_msg:
                         name, err = err_msg.split('|')
                         if 'SchainWarning' in err:
                             log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
                         elif 'SchainError' in err:
                             log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
                         else:
                             log.error(err, name)
                     else:
                         name, err = self.name, err_msg
                     time.sleep(1)
                     ctx.term()
                     message = ''.join(err)
                     if err_msg:
                         subject = 'SChain v%s: Error running %s\n' % (
                             schainpy.__version__, self.name)
                         subtitle = 'Hostname: %s\n' % socket.gethostbyname(
                             socket.gethostname())
                         subtitle += 'Working directory: %s\n' % os.path.abspath('./')
                         subtitle += 'Configuration file: %s\n' % self.filename
                         subtitle += 'Time: %s\n' % str(datetime.datetime.now())
                         readUnitConfObj = self.getReadUnit()
                         if readUnitConfObj:
                             subtitle += '\nInput parameters:\n'
                             subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
                             subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
                             subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
                             subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
                             subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
                         a = Alarm(
                             modes=self.alarm,
                             email=self.email,
                             message=message,
                             subject=subject,
                             subtitle=subtitle,
                             filename=self.filename
                         )
                         a.start()
                 def setFilename(self, filename):
                     self.filename = filename
                 def runProcs(self):
                     err = False
                     n = len(self.configurations)
                     while not err:
                         #print("STAR")
                         for conf in self.getUnits():
                             #print("CONF: ",conf)
                             ok = conf.run()
                             if ok == 'Error':
+                                #self.removeProcUnit(conf.id) #remove proc Unit
                                 n -= 1
                                 continue
                             elif not ok:
                                 break
                         if n == 0:
                             err = True
                 def run(self):
                     log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
                     self.started = True
                     self.start_time = time.time()
                     self.createObjects()
                     self.runProcs()
                     log.success('{} Done (Time: {:4.2f}s)'.format(
                         self.name,
                         time.time()-self.start_time), '')

schainpy/model/io/jroIO_param.py +2 0

             import os
             import time
             import datetime
             import numpy
             import h5py
             import schainpy.admin
             from schainpy.model.data.jrodata import *
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.io.jroIO_base import *
             from schainpy.utils import log
             class HDFReader(Reader, ProcessingUnit):
                 """Processing unit to read HDF5 format files
                 This unit reads HDF5 files created with `HDFWriter` operation contains
                 by default two groups Data and Metadata all variables would be saved as `dataOut`
                 attributes.
                 It is possible to read any HDF5 file by given the structure in the `description`
                 parameter, also you can add extra values to metadata with the parameter `extras`.
                 Parameters:
                 -----------
                 path : str
                     Path where files are located.
                 startDate : date
                     Start date of the files
                 endDate : list
                     End date of the files
                 startTime : time
                     Start time of the files
                 endTime : time
                     End time of the files
                 description : dict, optional
                     Dictionary with the description of the HDF5 file
                 extras : dict, optional
                     Dictionary with extra metadata to be be added to `dataOut`
                 Examples
                 --------
                 desc = {
                     'Data': {
                         'data_output': ['u', 'v', 'w'],
                         'utctime': 'timestamps',
                     }  ,
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 desc = {
                     'Data': {
                         'data_output': 'winds',
                         'utctime': 'timestamps'
                     },
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 extras = {
                     'timeZone': 300
                 }
                 reader = project.addReadUnit(
                     name='HDFReader',
                     path='/path/to/files',
                     startDate='2019/01/01',
                     endDate='2019/01/31',
                     startTime='00:00:00',
                     endTime='23:59:59',
                     # description=json.dumps(desc),
                     # extras=json.dumps(extras),
                     )
                 """
                 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.ext = ".hdf5"
                     self.optchar = "D"
                     self.meta = {}
                     self.data = {}
                     self.open_file = h5py.File
                     self.open_mode = 'r'
                     self.description = {}
                     self.extras = {}
                     self.filefmt = "*%Y%j***"
                     self.folderfmt = "*%Y%j"
                     self.utcoffset = 0
                     self.dataOut = Parameters()
                     self.dataOut.error=False            ## NOTE: Importante definir esto antes inicio
                     self.dataOut.flagNoData = True
                 def setup(self, **kwargs):
                     self.set_kwargs(**kwargs)
                     if not self.ext.startswith('.'):
                         self.ext = '.{}'.format(self.ext)
                     if self.online:
                         log.log("Searching files in online mode...", self.name)
                         for nTries in range(self.nTries):
                             fullpath = self.searchFilesOnLine(self.path, self.startDate,
                                 self.endDate, self.expLabel, self.ext, self.walk,
                                 self.filefmt, self.folderfmt)
                             pathname, filename = os.path.split(fullpath)
                             try:
                                 fullpath = next(fullpath)
                             except:
                                 fullpath = None
                             if fullpath:
                                 break
                             log.warning(
                                 'Waiting {} sec for a valid file in {}: try {} ...'.format(
                                     self.delay, self.path, nTries + 1),
                                 self.name)
                             time.sleep(self.delay)
                         if not(fullpath):
                             raise schainpy.admin.SchainError(
                                 'There isn\'t any valid file in {}'.format(self.path))
                         pathname, filename = os.path.split(fullpath)
                         self.year = int(filename[1:5])
                         self.doy = int(filename[5:8])
                         self.set = int(filename[8:11]) - 1
                     else:
                         log.log("Searching files in {}".format(self.path), self.name)
                         self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
                             self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
                     self.setNextFile()
                 def readFirstHeader(self):
                     '''Read metadata and data'''
                     self.__readMetadata()
                     self.__readData()
                     self.__setBlockList()
                     for attr in self.meta:
                         setattr(self.dataOut, attr, self.meta[attr])
                     self.blockIndex = 0
                     return
                 def __setBlockList(self):
                     '''
                     Selects the data within the times defined
                     self.fp
                     self.startTime
                     self.endTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     startTime = self.startTime
                     endTime = self.endTime
                     thisUtcTime = self.data['utctime'] + self.utcoffset
                     self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
                     thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
                     self.startFileDatetime = thisDatetime
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     self.blocksPerFile = len(thisUtcTime)
                     return
                 def __readMetadata(self):
                     '''
                     Reads Metadata
                     '''
                     meta = {}
                     if self.description:
                         for key, value in self.description['Metadata'].items():
                             meta[key] = self.fp[value][()]
                     else:
                         grp = self.fp['Metadata']
                         for name in grp:
                             meta[name] = grp[name][()]
                     if self.extras:
                         for key, value in self.extras.items():
                             meta[key] = value
                     self.meta = meta
                     return
                 def checkForRealPath(self, nextFile, nextDay):
                     # print("check FRP")
                     # dt = self.startFileDatetime + datetime.timedelta(1)
                     # filename = '{}.{}{}'.format(self.path, dt.strftime('%Y%m%d'), self.ext)
                     # fullfilename = os.path.join(self.path, filename)
                     # print("check Path ",fullfilename,filename)
                     # if os.path.exists(fullfilename):
                     #     return fullfilename, filename
                     # return None, filename
                     return None,None
                 def __readData(self):
                     data = {}
                     if self.description:
                         for key, value in self.description['Data'].items():
                             if isinstance(value, str):
                                 if isinstance(self.fp[value], h5py.Dataset):
                                     data[key] = self.fp[value][()]
                                 elif isinstance(self.fp[value], h5py.Group):
                                     array = []
                                     for ch in self.fp[value]:
                                         array.append(self.fp[value][ch][()])
                                     data[key] = numpy.array(array)
                             elif isinstance(value, list):
                                 array = []
                                 for ch in value:
                                     array.append(self.fp[ch][()])
                                 data[key] = numpy.array(array)
                     else:
                         grp = self.fp['Data']
                         for name in grp:
                             if isinstance(grp[name], h5py.Dataset):
                                 array = grp[name][()]
                             elif isinstance(grp[name], h5py.Group):
                                 array = []
                                 for ch in grp[name]:
                                     array.append(grp[name][ch][()])
                                 array = numpy.array(array)
                             else:
                                 log.warning('Unknown type: {}'.format(name))
                             if name in self.description:
                                 key = self.description[name]
                             else:
                                 key = name
                             data[key] = array
                     self.data = data
                     return
                 def getData(self):
                     if not self.isDateTimeInRange(self.startFileDatetime, self.startDate, self.endDate, self.startTime, self.endTime):
                         self.dataOut.flagNoData = True
                         self.blockIndex = self.blocksPerFile
                         self.dataOut.error = True     # TERMINA EL PROGRAMA
                         return
                     for attr in self.data:
                         if self.data[attr].ndim == 1:
                             setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
                         else:
                             setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
                     self.blockIndex += 1
                     if self.blockIndex == 1:
                         log.log("Block No. {}/{} -> {}".format(
                             self.blockIndex,
                             self.blocksPerFile,
                             self.dataOut.datatime.ctime()), self.name)
                     else:
                         log.log("Block No. {}/{} ".format(
                             self.blockIndex,
                             self.blocksPerFile),self.name)
                     if self.blockIndex == self.blocksPerFile:
                         self.setNextFile()
                     self.dataOut.flagNoData = False
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             #@MPDecorator
             class HDFWrite(Operation):
                 """Operation to write HDF5 files.
                 The HDF5 file contains by default two groups Data and Metadata where
                 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
                 parameters, data attributes are normaly time dependent where the metadata
                 are not.
                 It is possible to customize the structure of the HDF5 file with the
                 optional description parameter see the examples.
                 Parameters:
                 -----------
                 path : str
                     Path where files will be saved.
                 blocksPerFile : int
                     Number of blocks per file
                 metadataList : list
                     List of the dataOut attributes that will be saved as metadata
                 dataList : int
                     List of the dataOut attributes that will be saved as data
                 setType : bool
                     If True the name of the files corresponds to the timestamp of the data
                 description : dict, optional
                     Dictionary with the desired description of the HDF5 file
                 Examples
                 --------
                 desc = {
                     'data_output': {'winds': ['z', 'w', 'v']},
                     'utctime': 'timestamps',
                     'heightList': 'heights'
                 }
                 desc = {
                     'data_output': ['z', 'w', 'v'],
                     'utctime': 'timestamps',
                     'heightList': 'heights'
                 }
                 desc = {
                     'Data': {
                         'data_output': 'winds',
                         'utctime': 'timestamps'
                     },
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 writer = proc_unit.addOperation(name='HDFWriter')
                 writer.addParameter(name='path', value='/path/to/file')
                 writer.addParameter(name='blocksPerFile', value='32')
                 writer.addParameter(name='metadataList', value='heightList,timeZone')
                 writer.addParameter(name='dataList',value='data_output,utctime')
                 # writer.addParameter(name='description',value=json.dumps(desc))
                 """
                 ext = ".hdf5"
                 optchar = "D"
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None                  #eval ??????
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 currentDay = None
                 lastTime = None
                 timeZone = "ut"
                 hourLimit = 3
                 breakDays = True
                 def __init__(self):
                     Operation.__init__(self)
                 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None,
                             description={},timeZone = "ut",hourLimit = 3, breakDays=True):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataList = [s.strip() for s in dataList]
                     self.setType = setType
                     self.description = description
                     self.timeZone = timeZone
                     self.hourLimit = hourLimit
                     self.breakDays = breakDays
                     if self.metadataList is None:
                         self.metadataList = self.dataOut.metadata_list
                     tableList = []
                     dsList = []
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         if hasattr(self.dataOut, self.dataList[i]):
                             dataAux = getattr(self.dataOut, self.dataList[i])
                             dsDict['variable'] = self.dataList[i]
                         else:
                             log.warning('Attribute {} not found in dataOut', self.name)
                             continue
                         if dataAux is None:
                             continue
                         elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
                             dsDict['nDim'] = 0
                         else:
                             dsDict['nDim'] = len(dataAux.shape)
                             dsDict['shape'] = dataAux.shape
                             dsDict['dsNumber'] = dataAux.shape[0]
                             dsDict['dtype'] = dataAux.dtype
                         dsList.append(dsDict)
                     self.blockIndex = 0
                     self.dsList = dsList
                     self.currentDay = self.dataOut.datatime.date()
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     timeTuple = None
                     if self.timeZone == "lt":
                         timeTuple = time.localtime(currentTime)
                     else :
                         timeTuple = time.gmtime(currentTime)
                     dataDay = timeTuple.tm_yday
                     if self.lastTime is None:
                         self.lastTime = currentTime
                         self.currentDay = dataDay
                         return False
                     timeDiff = currentTime - self.lastTime
                     #Si el dia es diferente o si la diferencia entre un dato y otro supera self.hourLimit
                     if (dataDay != self.currentDay) and self.breakDays:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > self.hourLimit*60*60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def run(self, dataOut,**kwargs):
                     self.dataOut = dataOut
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return self.dataOut
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = None
                     if self.timeZone == "lt":
                         timeTuple = time.localtime(self.dataOut.utctime)
                     elif self.timeZone == "ut":
                         timeTuple = time.gmtime(self.dataOut.utctime)
                     #print("path: ",timeTuple)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join(path, subfolder)
                     if os.path.exists(fullpath):
                         filesList = os.listdir(fullpath)
                         filesList = [k for k in filesList if k.startswith(self.optchar)]
                         if len( filesList ) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     self.filename = os.path.join( path, subfolder, file )
                 def getLabel(self, name, x=None):
                     if x is None:
                         if 'Data' in self.description:
                             data = self.description['Data']
                             if 'Metadata' in self.description:
                                 data.update(self.description['Metadata'])
                         else:
                             data = self.description
                         if name in data:
                             if isinstance(data[name], str):
                                 return data[name]
                             elif isinstance(data[name], list):
                                 return None
                             elif isinstance(data[name], dict):
                                 for key, value in data[name].items():
                                     return key
                         return name
                     else:
                         if 'Metadata' in self.description:
                             meta = self.description['Metadata']
                         else:
                             meta = self.description
                         if name in meta:
                             if isinstance(meta[name], list):
                                 return meta[name][x]
                             elif isinstance(meta[name], dict):
                                 for key, value in meta[name].items():
                                     return value[x]
                         if 'cspc' in name:
                             return 'pair{:02d}'.format(x)
                         else:
                             return 'channel{:02d}'.format(x)
                 def writeMetadata(self, fp):
                     if self.description:
                         if 'Metadata' in self.description:
                             grp = fp.create_group('Metadata')
                         else:
                             grp = fp
                     else:
                         grp = fp.create_group('Metadata')
                     for i in range(len(self.metadataList)):
                         if not hasattr(self.dataOut, self.metadataList[i]):
                             log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
                             continue
                         value = getattr(self.dataOut, self.metadataList[i])
                         if isinstance(value, bool):
                             if value is True:
                                 value = 1
                             else:
                                 value = 0
                         grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
                     return
                 def writeData(self, fp):
                     if self.description:
                         if 'Data' in self.description:
                             grp = fp.create_group('Data')
                         else:
                             grp = fp
                     else:
                         grp = fp.create_group('Data')
                     dtsets = []
                     data = []
                     for dsInfo in self.dsList:
                         if dsInfo['nDim'] == 0:
                             ds = grp.create_dataset(
                                 self.getLabel(dsInfo['variable']),
                                 (self.blocksPerFile, ),
                                 chunks=True,
                                 dtype=numpy.float64)
                             dtsets.append(ds)
                             data.append((dsInfo['variable'], -1))
                         else:
                             label = self.getLabel(dsInfo['variable'])
                             if label is not None:
                                 sgrp = grp.create_group(label)
                             else:
                                 sgrp = grp
                             for i in range(dsInfo['dsNumber']):
                                 ds = sgrp.create_dataset(
                                     self.getLabel(dsInfo['variable'], i),
                                     (self.blocksPerFile, ) + dsInfo['shape'][1:],
                                     chunks=True,
                                     dtype=dsInfo['dtype'])
                                 dtsets.append(ds)
                                 data.append((dsInfo['variable'], i))
                     fp.flush()
                     log.log('Creating file: {}'.format(fp.filename), self.name)
                     self.ds = dtsets
                     self.data = data
                     self.firsttime = True
                     return
                 def putData(self):
                     if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
                         self.closeFile()
                         self.setNextFile()
                         self.dataOut.flagNoData = False
                         self.blockIndex = 0
                         return
                     if self.blockIndex == 0:
                         #Escribir metadata Aqui???
                         #Setting HDF5 File
                         self.fp = h5py.File(self.filename, 'w')
                         #write metadata
                         self.writeMetadata(self.fp)
                         #Write data
                         self.writeData(self.fp)
                         log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
+                    elif (self.blockIndex % 10 ==0):
+                        log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
                     else:
                         log.log('Block No. {}/{}'.format(self.blockIndex+1, self.blocksPerFile), self.name)
                     for i, ds in enumerate(self.ds):
                         attr, ch = self.data[i]
                         if ch == -1:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)
                         else:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
                     self.blockIndex += 1
                     self.fp.flush()
                     self.dataOut.flagNoData = True
                 def closeFile(self):
                     if self.blockIndex != self.blocksPerFile:
                         for ds in self.ds:
                             ds.resize(self.blockIndex, axis=0)
                     if self.fp:
                         self.fp.flush()
                         self.fp.close()
                 def close(self):
                     self.closeFile()

schainpy/model/utils/jroutils_ftp.py +99 -10

             '''
             @author: Daniel Suarez
             '''
             import os
             import glob
             import ftplib
             try:
                 import paramiko
                 import scp
             except:
                 pass
             import time
             import threading
             Thread = threading.Thread
             # try:
             #     from gevent import sleep
             # except:
             from time import sleep
             from schainpy.model.data.jrodata import *
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             #@MPDecorator
             class Remote(Thread):
                 """
                 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
                 used to upload or download files remotely.
                 Non-standard Python modules used:
                     None
                 Written by:
                     "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
                 Modified by:
                     -
                 """
                 server = None
                 username = None
                 password = None
                 remotefolder = None
                 key_filename=None
                 period = 60
                 fileList = []
                 bussy = False
                 def __init__(self, server, username, password, remotefolder, period=60,key_filename=None):
                     Thread.__init__(self)
                     self.setDaemon(True)
                     self.status = 0
                     self.__server = server
                     self.__username = username
                     self.__password = password
                     self.__remotefolder = remotefolder
                     self.period = period
                     self.key_filename = key_filename
                     self.fileList = []
                     self.bussy = False
                     self.stopFlag = False
                     print("[Remote Server] Opening server: %s" %self.__server)
                     if self.open(self.__server, self.__username, self.__password, self.__remotefolder,key_filename=self.key_filename):
                         print("[Remote Server] %s server was opened successfully" %self.__server)
                     #self.close()
                     self.mutex = threading.Lock()
                 def stop(self):
                     self.stopFlag = True
                     self.join(10)
                 def open(self):
                     """
                     Connect to server and create a connection class (FTP or SSH) to remote server.
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def close(self):
                     """
                     Close connection to server
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def mkdir(self, remotefolder):
                     """
                     Create a folder remotely
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def cd(self, remotefolder):
                     """
                     Change working directory in remote server
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def download(self, filename, localfolder=None):
                     """
                     Download a file from server to local host
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def sendFile(self, fullfilename):
                     """
                     sendFile method is used to upload a local file to the current directory in remote server
                     Inputs:
                         fullfilename        - full path name of local file to store in remote directory
                     Returns:
 in error case else 1
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def upload(self, fullfilename, remotefolder=None):
                     """
                     upload method is used to upload a local file to remote directory. This method changes
                     working directory before sending a file.
                     Inputs:
                         fullfilename        - full path name of local file to store in remote directory
                         remotefolder    - remote directory
                     Returns:
 in error case else 1
                     """
                     print("[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder))
                     if not self.status:
                         return 0
                     if remotefolder == None:
                         remotefolder = self.remotefolder
                     if not self.cd(remotefolder):
                         return 0
                     if not self.sendFile(fullfilename):
                         print("[Remote Server] Error uploading file %s" %fullfilename)
                         return 0
                     return 1
                 def delete(self, filename):
                     """
                     Remove a file from remote server
                     """
                     pass
                 def updateFileList(self, fileList):
                     """
                     Remove a file from remote server
                     """
                     if fileList == self.fileList:
                         return 0
                     self.mutex.acquire()
                     #         init = time.time()
                     #
                     #         while(self.bussy):
                     #             sleep(0.1)
                     #             if time.time() - init > 2*self.period:
                     #                 return 0
                     self.fileList = fileList
                     self.mutex.release()
                     return 1
                 def run(self):
                     if not self.status:
                         print("Finishing FTP service")
                         return
                     if not self.cd(self.remotefolder):
                         raise ValueError("Could not access to the new remote directory: %s" %self.remotefolder)
                     while True:
                         for i in range(self.period):
                             if self.stopFlag:
                                 break
                             sleep(1)
                         if self.stopFlag:
                             break
                         #   self.bussy = True
                         self.mutex.acquire()
                         print("[Remote Server] Opening %s" %self.__server)
                         if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
                             self.mutex.release()
                             continue
                         for thisFile in self.fileList:
                             self.upload(thisFile, self.remotefolder)
                         print("[Remote Server] Closing %s" %self.__server)
                         self.close()
                         self.mutex.release()
                         # self.bussy = False
                     print("[Remote Server] Thread stopped successfully")
             class FTPClient(Remote):
                 __ftpClientObj = None
                 def __init__(self, server, username, password, remotefolder, period=60):
                     """
                     """
                     Remote.__init__(self, server, username, password, remotefolder, period)
                 def open(self, server, username, password, remotefolder):
                     """
                     This method is used to set FTP parameters and establish a connection to remote server
                     Inputs:
                         server    - remote server IP Address
                         username    - remote server Username
                         password    - remote server password
                         remotefolder    - remote server current working directory
                     Return:
                         Boolean     -     Returns 1 if a connection has been established, 0 otherwise
                     Affects:
                         self.status        - in case of error or fail connection this parameter is set to 0 else 1
                     """
                     if server == None:
                         raise ValueError("FTP server should be defined")
                     if username == None:
                         raise ValueError("FTP username should be defined")
                     if password == None:
                         raise ValueError("FTP password should be defined")
                     if remotefolder == None:
                         raise ValueError("FTP remote folder should be defined")
                     try:
                         ftpClientObj = ftplib.FTP(server)
                     except ftplib.all_errors as e:
                         print("[FTP Server]: FTP server connection fail: %s" %server)
                         print("[FTP Server]:", e)
                         self.status = 0
                         return 0
                     try:
                         ftpClientObj.login(username, password)
                     except ftplib.all_errors:
                         print("[FTP Server]: FTP username or password are incorrect")
                         self.status = 0
                         return 0
                     if remotefolder == None:
                         remotefolder = ftpClientObj.pwd()
                     else:
                         try:
                             ftpClientObj.cwd(remotefolder)
                         except ftplib.all_errors:
                             print("[FTP Server]: FTP remote folder is invalid: %s" %remotefolder)
                             remotefolder = ftpClientObj.pwd()
                     self.server = server
                     self.username = username
                     self.password = password
                     self.remotefolder = remotefolder
                     self.__ftpClientObj = ftpClientObj
                     self.status = 1
                     return 1
                 def close(self):
                     """
                     Close connection to remote server
                     """
                     if not self.status:
                         return 0
                     self.__ftpClientObj.close()
                 def mkdir(self, remotefolder):
                     """
                     mkdir is used to make a new directory in remote server
                     Input:
                         remotefolder    - directory name
                     Return:
 in error case else 1
                     """
                     if not self.status:
                         return 0
                     try:
                         self.__ftpClientObj.mkd(dirname)
                     except ftplib.all_errors:
                         print("[FTP Server]: Error creating remote folder: %s" %remotefolder)
                         return 0
                     return 1
                 def cd(self, remotefolder):
                     """
                     cd is used to change remote working directory on server
                     Input:
                         remotefolder    - current working directory
                     Affects:
                         self.remotefolder
                     Return:
 in case of error else 1
                     """
                     if not self.status:
                         return 0
                     if remotefolder == self.remotefolder:
                         return 1
                     try:
                         self.__ftpClientObj.cwd(remotefolder)
                     except ftplib.all_errors:
                         print('[FTP Server]: Error changing to %s' %remotefolder)
                         print('[FTP Server]: Trying to create remote folder')
                         if not self.mkdir(remotefolder):
                             print('[FTP Server]: Remote folder could not be created')
                             return 0
                         try:
                             self.__ftpClientObj.cwd(remotefolder)
                         except ftplib.all_errors:
                             return 0
                     self.remotefolder = remotefolder
                     return 1
                 def sendFile(self, fullfilename):
                     if not self.status:
                         return 0
                     fp = open(fullfilename, 'rb')
                     filename = os.path.basename(fullfilename)
                     command = "STOR %s" %filename
                     try:
                         self.__ftpClientObj.storbinary(command, fp)
                     except ftplib.all_errors as e:
                         print("[FTP Server]:", e)
                         return 0
                     try:
                         self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
                     except ftplib.all_errors as e:
                         print("[FTP Server]:", e)
                     fp.close()
                     return 1
             class SSHClient(Remote):
                 __sshClientObj = None
                 __scpClientObj = None
                 def __init__(self, server, username, password, remotefolder, period=60,key_filename=None):
                     """
                     """
                     Remote.__init__(self, server, username, password, remotefolder, period, key_filename)
                 def open(self, server, username, password, remotefolder, port=22, key_filename=None):
                     """
                         This method is used to set SSH parameters and establish a connection to a remote server
                         Inputs:
                             server    - remote server IP Address
                             username    - remote server Username
                             password    - remote server password
                             remotefolder    - remote server current working directory
                             key_filename    -   filename of the private key/optional
                         Return: void
                         Affects:
                             self.status        - in case of error or fail connection this parameter is set to 0 else 1
                     """
                     #import socket
                     if server == None:
                         raise ValueError("SSH server should be defined")
                     if username == None:
                         raise ValueError("SSH username should be defined")
                     if password == None:
                         raise ValueError("SSH password should be defined")
                     if remotefolder == None:
                         raise ValueError("SSH remote folder should be defined")
                     self.__sshClientObj = paramiko.SSHClient()
                     self.__sshClientObj.load_system_host_keys()
                     self.__sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
                     self.status = 0
                     try:
                         if key_filename != None:
                             self.__sshClientObj.connect(server, username=username, password=password, port=port, key_filename=key_filename)
                         else:
                             self.__sshClientObj.connect(server, username=username, password=password, port=port)
                     except paramiko.AuthenticationException as e:
                 #             print "SSH username or password are incorrect: %s"
                         print("[SSH Server]:", e)
                         return 0
                     # except SSHException as e:
                     #     print("[SSH Server]:", e)
                     #     return 0
                     # except socket.error:
                     #     self.status = 0
                     #     print("[SSH Server]:", e)
                     #     return 0
                     self.status = 1
                     #self.__scpClientObj = scp.SCPClient(self.__sshClientObj.get_transport(), socket_timeout=30)
                     self.__scpClientObj = self.__sshClientObj.open_sftp()
                     if remotefolder == None:
                         remotefolder = self.pwd()
                     self.server = server
                     self.username = username
                     self.password = password
                     # self.__sshClientObj = self.__sshClientObj
                     # self.__scpClientObj = self.__scpClientObj
                     self.status = 1
                     if not self.cd(remotefolder):
                         raise ValueError("[SSH Server]: Could not access to remote folder: %s" %remotefolder)
                         return 0
                     self.remotefolder = remotefolder
                     return 1
                 def close(self):
                     """
                         Close connection to remote server
                     """
                     if not self.status:
                         return 0
                     self.__scpClientObj.close()
                     self.__sshClientObj.close()
                 def __execute(self, command):
                     """
                         __execute a command on remote server
                         Input:
                             command    - Exmaple 'ls -l'
                         Return:
 in error case else 1
                     """
                     if not self.status:
                         return 0
                     stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
                     result = stderr.readlines()
                     if len(result) > 1:
                         return 0
                     result = stdout.readlines()
                     if len(result) > 1:
                         return result[0][:-1]
                     return 1
                 def mkdir(self, remotefolder):
                     """
                         mkdir is used to make a new directory in remote server
                         Input:
                             remotefolder    - directory name
                         Return:
 in error case else 1
                     """
                     command = 'mkdir %s' %remotefolder
                     return self.__execute(command)
                 def pwd(self):
                     command = 'pwd'
                     return self.__execute(command)
                 def cd(self, remotefolder):
                     """
                         cd is used to change remote working directory on server
                         Input:
                             remotefolder    - current working directory
                         Affects:
                             self.remotefolder
                         Return:
 in case of error else 1
                     """
                     if not self.status:
                         return 0
                     if remotefolder == self.remotefolder:
                         return 1
                     chk_command = "cd %s; pwd" %remotefolder
                     mkdir_command = "mkdir %s" %remotefolder
                     if not self.__execute(chk_command):
                         if not self.__execute(mkdir_command):
                             self.remotefolder = None
                             return 0
                     self.remotefolder = remotefolder
                     return 1
                 def sendFile(self, fullfilename):
                     if not self.status:
                         return 0
                     remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
                     print("remotefile",fullfilename, remotefile)
                     try:
                         self.__scpClientObj.put(fullfilename,remotefile)
                     except  paramiko.SSHException as e:
                         print("[SSH Server]", str(e))
                         print(fullfilename," to ",remotefile)
                         return 0
                     #command = 'chmod 775 %s' %remotefile
                     return 1#self.__execute(command)
             #@MPDecorator
             class SendToServerProc(ProcessingUnit):
                 sendByTrigger = False
                 def __init__(self, **kwargs):
                     ProcessingUnit.__init__(self)
                     self.isConfig = False
                     self.clientObj = None
                     self.dataOut = Parameters()
                     self.dataOut.error=False
                     self.dataOut.flagNoData=True
                 def setup(self, server=None, username="", password="", remotefolder="", localfolder="",
                 ext='.png', period=60, protocol='ftp', sendByTrigger=False, key_filename=None):
                     self.server = server
                     self.username = username
                     self.password = password
                     self.remotefolder = remotefolder
                     self.clientObj = None
                     self.localfolder = localfolder
                     self.ext = ext
                     self.sendByTrigger = sendByTrigger
                     self.period = period
                     self.key_filename = key_filename
                     if self.sendByTrigger:
                         self.period = 1000000000000  #para que no se ejecute por tiempo
                     if str.lower(protocol) == 'ftp':
                         self.clientObj = FTPClient(server, username, password, remotefolder, period)
                     if str.lower(protocol) == 'ssh':
                         self.clientObj = SSHClient(self.server, self.username, self.password,
                          self.remotefolder, period=600000,key_filename=self.key_filename)
                     if not self.clientObj:
                         raise ValueError("%s has been chosen as remote access protocol but it is not valid" %protocol)
                     print("Send to Server setup complete")
                 def findFiles(self):
                     if not type(self.localfolder) == list:
                         folderList = [self.localfolder]
                     else:
                         folderList = self.localfolder
                     #Remove duplicate items
                     folderList = list(set(folderList))
                     fullfilenameList = []
                     for thisFolder in folderList:
                         print("[Remote Server]: Searching files on %s" %thisFolder)
                         filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
                         if len(filenameList) < 1:
                             continue
                         for thisFile in filenameList:
                             fullfilename = os.path.join(thisFolder, thisFile)
                             if fullfilename in fullfilenameList:
                                 continue
                             #Only files modified in the last 30 minutes are considered
                             if os.path.getmtime(fullfilename) < time.time() - 30*60:
                                 continue
                             fullfilenameList.append(fullfilename)
                             fullfilenameList.sort()
                     return fullfilenameList
                 def run(self, **kwargs):
                     if not self.isConfig:
                         self.init = time.time()
                         self.setup(**kwargs)
                         self.isConfig = True
                     if not self.clientObj.is_alive():
                         print("[Remote Server]: Restarting connection ")
                         self.setup( **kwargs)
                     if ((time.time() - self.init) >= self.period  and not self.sendByTrigger) or  (self.sendByTrigger and not self.dataIn.flagNoData):
                         fullfilenameList = self.findFiles()
                         if self.sendByTrigger:
                             if self.clientObj.upload(fullfilenameList[-1]): #last file to send
                                 print("[Remote Server] upload finished successfully")
                         else:
                             for file in fullfilenameList:
                                 self.clientObj.upload(file)
                         # if self.clientObj.updateFileList(fullfilenameList):
                         #     print("[Remote Server]: Sending the next files ", str(fullfilenameList))
                         self.init = time.time()
                 def close(self):
                     print("[Remote Server] Stopping thread")
                     self.clientObj.stop()
             class SendByRSYNCProc(ProcessingUnit):
                 sendByTrigger = False
                 def __init__(self, **kwargs):
                     ProcessingUnit.__init__(self)
                     self.isConfig = False
                     self.dataOut = Parameters()
                     self.dataOut.error=False
                     self.dataOut.flagNoData=True
                 def setup(self, server="", username="", remotefolder="", localfolder="",sendByTrigger=True,
                     period=60, key_filename=None, port=22 ,param1="", param2=""):
                     self.server = server
                     self.username = username
                     self.remotefolder = remotefolder
                     self.localfolder = localfolder
                     self.period = period
                     self.key_filename = key_filename
                     if type(param1)==str:
                         self.param1 = list(param1.split(","))
                     else:
                         self.param1 = param1
                     if type(param2)==str:
                         self.param2 = list(param2.split(","))
                     else:
                         self.param2 = param2
                     self.port = port
                     self.sendByTrigger = sendByTrigger
                     if self.sendByTrigger:
                         self.period = 1000000000000  #para que no se ejecute por tiempo
                     self.command ="rsync "
                 def syncFolders(self):
                     self.command ="rsync "
                     for p1 in self.param1:
                         self.command += " -"+str(p1)
                     for p2 in self.param2:
                         self.command += " --"+str(p2)
                     if self.key_filename != None:
                         self.command += """ "ssh -i {} -p {}" """.format(self.key_filename, self.port)
                     self.command += " {} ".format(self.localfolder)
                     self.command += " {}@{}:{}".format(self.username,self.server,self.remotefolder)
                     print("CMD: ",self.command)
                     #os.system(self.command)
                     return
                 def run(self, **kwargs):
                     if not self.isConfig:
                         self.init = time.time()
                         self.setup(**kwargs)
                         self.isConfig = True
                     if self.sendByTrigger and not self.dataIn.flagNoData:
                         self.syncFolders()
                     else:
                         if (time.time() - self.init) >= self.period:
                             self.syncFolders()
                             self.init = time.time()
                     return
             class FTP(object):
                 """
                 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
                 Non-standard Python modules used: None
                 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe  Oct. 26, 2010
+                Modified:
+                Joab Apaza  Feb. 2022
                 """
                 def __init__(self,server = None, username=None, password=None, remotefolder=None):
                     """
                     This method is used to setting parameters for FTP and establishing connection to remote server
                     Inputs:
                         server    - remote server IP Address
                         username    - remote server Username
                         password    - remote server password
                         remotefolder    - remote server current working directory
                     Return: void
                     Affects:
                         self.status    - in Error Case or Connection Failed this parameter is set to 1 else 0
                         self.folderList    - sub-folder list of remote folder
                         self.fileList    - file list of remote folder
                     """
                     if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
                         server, username, password, remotefolder = self.parmsByDefault()
                     self.server = server
                     self.username = username
                     self.password = password
                     self.remotefolder = remotefolder
                     self.file = None
                     self.ftp = None
                     self.status = 0
                     try:
                         self.ftp = ftplib.FTP(self.server)
                         self.ftp.login(self.username,self.password)
                         self.ftp.cwd(self.remotefolder)
                         #   print 'Connect to FTP Server: Successfully'
                     except ftplib.all_errors:
                         print('Error FTP Service')
                         self.status = 1
                         return
                     self.dirList = []
                     try:
                         self.dirList = self.ftp.nlst()
                     except ftplib.error_perm as resp:
                         if str(resp) == "550 No files found":
                                 print("no files in this directory")
                                 self.status = 1
                                 return
                     except ftplib.all_errors:
                         print('Error Displaying Dir-Files')
                         self.status = 1
                         return
                     self.fileList = []
                     self.folderList = []
                     #only for test
                     for f in self.dirList:
                         name, ext = os.path.splitext(f)
                         if ext != '':
                             self.fileList.append(f)
                 #                print 'filename: %s - size: %d'%(f,self.ftp.size(f))
                 def parmsByDefault(self):
                     server = 'jro-app.igp.gob.pe'
                     username = 'wmaster'
                     password = 'mst2010vhf'
                     remotefolder = '/home/wmaster/graficos'
                     return server, username, password, remotefolder
                 def mkd(self,dirname):
                     """
                     mkd is used to make directory in remote server
                     Input:
                         dirname    - directory name
                     Return:
 in error case else 0
                     """
                     try:
                         self.ftp.mkd(dirname)
                     except:
                         print('Error creating remote folder:%s'%dirname)
-                        return 1
+                        return False
-                    return 0
+                    return True
                 def delete(self,filename):
                     """
                     delete is used to delete file in current working directory of remote server
                     Input:
                         filename    - filename to delete in remote folder
                     Return:
 in error case else 0
                     """
                     try:
                         self.ftp.delete(filename)
                     except:
                         print('Error deleting remote file:%s'%filename)
-                        return 1
+                        return False
-                    return 0
+                    return True
                 def download(self,filename,localfolder):
                     """
                     download is used to downloading file from remote folder into local folder
                     Inputs:
                         filename    - filename to donwload
                         localfolder    - directory local to store filename
                     Returns:
                         self.status    - 1 in error case else 0
                     """
                     self.status = 0
                     if not(filename in self.fileList):
                         print('filename:%s not exists'%filename)
                         self.status = 1
                         return self.status
                     newfilename = os.path.join(localfolder,filename)
                     self.file = open(newfilename, 'wb')
                     try:
                         print('Download: ' + filename)
                         self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
                         print('Download Complete')
                     except ftplib.all_errors:
                         print('Error Downloading ' + filename)
                         self.status = 1
                         return self.status
                     self.file.close()
                     return self.status
                 def __handleDownload(self,block):
                     """
                     __handleDownload is used to handle writing file
                     """
                     self.file.write(block)
-                def upload(self,filename,remotefolder=None):
+                def upload(self,filename,remotefolder=None, mkdir=False):
                     """
-                    upload is used to uploading local file to remote directory
+                    upload is used to uploading local file to remote directory, and change the permission of the remote file
                     Inputs:
                         filename    - full path name of local file to store in remote directory
                         remotefolder    - remote directory
+                        mkdir         - if the remote folder doesn't exist, it will created
                     Returns:
                         self.status    - 1 in error case else 0
                     """
                     if remotefolder == None:
                         remotefolder = self.remotefolder
+                    if mkdir:
+                        if self.if_dir_exist(remotefolder):
+                            pass
+                        else:
+                            self.mkdir_r(remotefolder)
                     self.status = 0
                     try:
                         self.ftp.cwd(remotefolder)
                         self.file = open(filename, 'rb')
                         (head, tail) = os.path.split(filename)
                         command = "STOR " + tail
                         print('Uploading: ' + tail)
                         self.ftp.storbinary(command, self.file)
+                        print(self.cmd('SITE CHMOD 755 {}'.format(tail)))
                         print('Upload Completed')
                     except ftplib.all_errors:
                         print('Error Uploading ' + tail)
                         self.status = 1
                         return self.status
                     self.file.close()
                     #back to initial directory in __init__()
                     self.ftp.cwd(self.remotefolder)
                     return self.status
-                def dir(self,remotefolder):
+                def ch_dir(self,remotefolder):
                     """
-                    dir is used to change working directory of remote server and get folder and file list
+                    ch_dir is used to change working directory of remote server and get folder and file list
                     Input:
                         remotefolder    - current working directory
                     Affects:
                         self.fileList    - file list of working directory
                     Return:
                         infoList    - list with filenames and size of file in bytes
                         self.folderList    -    folder list
                     """
                     self.remotefolder = remotefolder
                     print('Change to ' + self.remotefolder)
                     try:
                         self.ftp.cwd(remotefolder)
                     except ftplib.all_errors:
                         print('Error Change to ' + self.remotefolder)
                         infoList = None
                         self.folderList = None
                         return infoList,self.folderList
                     self.dirList = []
                     try:
                         self.dirList = self.ftp.nlst()
                     except ftplib.error_perm as resp:
                         if str(resp) == "550 No files found":
                                 print("no files in this directory")
                                 infoList = None
                                 self.folderList = None
                                 return infoList,self.folderList
                     except ftplib.all_errors:
                         print('Error Displaying Dir-Files')
                         infoList = None
                         self.folderList = None
                         return infoList,self.folderList
                     infoList = []
                     self.fileList = []
                     self.folderList = []
                     for f in self.dirList:
                         name,ext = os.path.splitext(f)
                         if ext != '':
                             self.fileList.append(f)
                             value = (f,self.ftp.size(f))
                             infoList.append(value)
                         if ext == '':
                             self.folderList.append(f)
                     return infoList,self.folderList
                 def close(self):
                     """
                     close is used to close and end FTP connection
                     Inputs: None
                     Return: void
                     """
                     self.ftp.close()
+                def get_sub_dirs(self, path):
+                    """
+                    used internal
+                    Inputs:
+                        path    - path to split in sub folders
+                    Returns:
+                        sub_dirs    - list of sub folders
+                    """
+                    sub_dirs = path.split("/")
+                    if sub_dirs[0]=="/":
+                        sub_dirs.pop(0)
+                    if sub_dirs[-1]=="/":
+                        sub_dirs.pop(-1)
+                    return sub_dirs
+                def if_dir_exist(self,path):
+                    """
+                    check if a the path folder exists in the ftp server
+                    Inputs:
+                        path    - path to check
+                    Returns:
+                        status    - True if exists and False if it doesn't
+                    """
+                    sub_dirs = self.get_sub_dirs(path)
+                    main =  self.ftp.pwd()
+                    #print(main)
+                    for subdir in sub_dirs:
+                        folders = self.ftp.nlst(main)
+                        #print(folders)
+                        if (os.path.join(main,subdir) in folders):
+                            main = os.path.join(main,subdir)
+                            #print(main)
+                            continue
+                        else:
+                            return False
+                    return True
+                def cmd(self,command):
+                    """
+                    excecute a command in the FTP server
+                    """
+                    return self.ftp.sendcmd(command)
+                def mkdir_r(self,path):
+                    """
+                    create a remote folder and create sub folders if it is necessary
+                    Inputs:
+                        path    - path to create
+                    Returns:
+                        status    - True if succesfull else  False
+                    """
+                    sub_dirs = self.get_sub_dirs(path)
+                    main =  self.ftp.pwd()
+                    st = False
+                    #print(main)
+                    for subdir in sub_dirs:
+                        folders = self.ftp.nlst(main)
+                        #print(folders)
+                        folder = (os.path.join(main,subdir))
+                        if  (folder in folders):
+                            main = folder
+                            #print("new_main",main)
+                            continue
+                        else:
+                            print("creating...",folder)
+                            st = self.mkd(folder)
+                            print(self.cmd('SITE CHMOD 755 {}'.format(folder)))
+                            main = folder
+                    return st
             @MPDecorator
             class SendByFTP(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.status = 1
                     self.counter = 0
                 def error_print(self, ValueError):
                     print(ValueError, 'Error FTP')
                     print("don't worry the program is running...")
                 def worker_ftp(self, server, username, password, remotefolder, filenameList):
                     self.ftpClientObj = FTP(server, username, password, remotefolder)
                     for filename in filenameList:
                         self.ftpClientObj.upload(filename)
                     self.ftpClientObj.close()
                 def ftp_thread(self, server, username, password, remotefolder):
                     if not(self.status):
                         return
                     import multiprocessing
                     p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
                     p.start()
                     p.join(3)
                     if p.is_alive():
                         p.terminate()
                         p.join()
                         print('killing ftp process...')
                         self.status = 0
                         return
                     self.status = 1
                     return
                 def filterByExt(self, ext, localfolder):
                     fnameList = glob.glob1(localfolder,ext)
                     self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
                     if len(self.filenameList) == 0:
                         self.status = 0
                 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
                     self.counter += 1
                     if self.counter >= period:
                         self.filterByExt(ext, localfolder)
                         self.ftp_thread(server, username, password, remotefolder)
                         self.counter = 0
                     self.status = 1

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages