schain Commit - r1022:0af466e13e36 · Jicamarca Repository

fix merge conflicts

Juan C. Espinoza -

r1022:0af466e13e36

parent child

Context file:

r1022:0af466e13e36

Collapse all files

schainpy/scripts/optimun_offset.py created 644 +113 0

@@ -0,0 +1,113
	1	import h5py
	2	import numpy
	3	import matplotlib.pyplot as plt
	4	import glob
	5	import os
	6
	7	#---------------------- Functions ---------------------
	8
	9	def findFiles(path):
	10
	11	dirList = []
	12	fileList = []
	13
	14	for thisPath in os.listdir(path):
	15	dirList.append(os.path.join(path,thisPath))
	16	dirList.sort()
	17
	18	for thisDirectory in dirList:
	19	files = glob.glob1(thisDirectory, "*.hdf5")
	20	files.sort()
	21	for thisFile in files:
	22	fileList.append(os.path.join(thisDirectory,thisFile))
	23
	24	return fileList
	25
	26	def readFiles(fileList):
	27
	28	meteors_array = numpy.zeros((1,4))
	29
	30	for thisFile in fileList:
	31
	32	#Leer
	33	f1 = h5py.File(thisFile,'r')
	34	grp1 = f1['Data']
	35	grp2 = grp1['data_output']
	36	meteors1 = grp2['table0'][:]
	37	meteors_array = numpy.vstack((meteors_array,meteors1))
	38	#cerrar
	39	f1.close()
	40
	41	meteors_array = numpy.delete(meteors_array, 0, axis=0)
	42	meteors_list = [meteors_array[:,0],meteors_array[:,1],meteors_array[:,2],meteors_array[:,3]]
	43	return meteors_list
	44
	45	def estimateMean(offset_list):
	46
	47	mean_off = []
	48	axisY_off = []
	49	axisX_off = []
	50
	51	for thisOffset in offset_list:
	52	mean_aux = numpy.mean(thisOffset, axis = 0)
	53	mean_off.append(mean_aux)
	54	axisX_off.append(numpy.array([0,numpy.size(thisOffset)]))
	55	axisY_off.append(numpy.array([mean_aux,mean_aux]))
	56
	57	return mean_off, axisY_off, axisX_off
	58
	59	def plotPhases(offset0, axisY0, axisX0, title):
	60	f, axarr = plt.subplots(4, sharey=True)
	61	color = ['b','g','r','c']
	62	# plt.grid()
	63	for i in range(len(offset0)):
	64	thisMeteor = offset0[i]
	65	thisY = axisY0[i]
	66	thisX = axisX0[i]
	67	thisColor = color[i]
	68
	69	opt = thisColor + 'o'
	70	axarr[i].plot(thisMeteor,opt)
	71	axarr[i].plot(thisX, thisY, thisColor)
	72	axarr[i].set_ylabel('Offset ' + str(i))
	73
	74	plt.ylim((-180,180))
	75	axarr[0].set_title(title + ' Offsets')
	76	axarr[3].set_xlabel('Number of estimations')
	77
	78	return
	79
	80	def filterOffsets(offsets0, stdvLimit):
	81	offsets1 = []
	82
	83	for thisOffset in offsets0:
	84	pstd = numpy.std(thisOffset)*stdvLimit
	85	pmean = numpy.mean(thisOffset)
	86	outlier1 = thisOffset > pmean - pstd
	87	outlier2 = thisOffset < pmean + pstd
	88	not_outlier = numpy.logical_and(outlier1,outlier2)
	89	thisOffset1 = thisOffset[not_outlier]
	90	offsets1.append(thisOffset1)
	91
	92	return offsets1
	93
	94	#---------------------- Setup ---------------------------
	95
	96	path = '/home/jespinoza/Pictures/JASMET30/201608/phase'
	97	stdvLimit = 0.5
	98
	99	#---------------------- Script ---------------------------
	100
	101	fileList = findFiles(path)
	102	offsets0 = readFiles(fileList)
	103	mean0, axisY0, axisX0 = estimateMean(offsets0)
	104	plotPhases(offsets0, axisY0, axisX0, 'Original')
	105
	106	offsets1 = filterOffsets(offsets0, stdvLimit)
	107	mean1, axisY1, axisX1 = estimateMean(offsets1)
	108	plotPhases(offsets1, axisY1, axisX1, 'Filtered')
	109
	110	print "Original Offsets: %.2f, %.2f, %.2f, %.2f" % (mean0[0],mean0[1],mean0[2],mean0[3])
	111	print "Filtered Offsets: %.2f, %.2f, %.2f, %.2f" % (mean1[0],mean1[1],mean1[2],mean1[3])
	112
	113	plt.show()

.gitignore +6 -1

             # Byte-compiled / optimized / DLL files
             __pycache__/
             *.py[cod]
             *$py.class
             # C extensions
             *.so
             # Distribution / packaging
             .Python
             env/
             build/
             develop-eggs/
             dist/
             downloads/
             eggs/
             .eggs/
             lib/
             lib64/
             parts/
             sdist/
             var/
             wheels/
             *.egg-info/
             .installed.cfg
             *.egg
             # PyInstaller
             #  Usually these files are written by a python script from a template
             #  before PyInstaller builds the exe, so as to inject date/other infos into it.
             *.manifest
             *.spec
             # Installer logs
             pip-log.txt
             pip-delete-this-directory.txt
             # Unit test / coverage reports
             htmlcov/
             .tox/
             .coverage
             .coverage.*
             .cache
             nosetests.xml
             coverage.xml
             *,cover
             .hypothesis/
             # Translations
             *.mo
             *.pot
             # Django stuff:
             *.log
             local_settings.py
             # Flask stuff:
             instance/
             .webassets-cache
             # Scrapy stuff:
             .scrapy
             # Sphinx documentation
             docs/_build/
             # PyBuilder
             target/
             # Jupyter Notebook
             .ipynb_checkpoints
             # pyenv
             .python-version
             # celery beat schedule file
             celerybeat-schedule
             # SageMath parsed files
             *.sage.py
             # dotenv
             .env
             # virtualenv
             .venv
             venv/
             ENV/
             # Spyder project settings
             .spyderproject
             .spyproject
             # Rope project settings
             .ropeproject
             # mkdocs documentation
             /site
             # eclipse
             .project
             .pydevproject
             # vscode
             .vscode
-            schainpy/scripts/
  No newline at end of file
+            .svn/
+            *.png
+            *.pyc
+            schainpy/scripts
+            schaingui/node_modules

README.md +1 -1

             # Signal Chain
             ## Introduction
             Signal Chain (SCh) is a radar data processing library developed using [Python](www.python.org) at JRO. SCh provides modules to read, write, process and plot data.
             ## Installation
             Install system dependencies, clone the latest version from [git](http://jro-dev.igp.gob.pe/rhodecode/schain/) and install it as a normal python package.
             ```
             $ sudo apt-get install python-pip python-dev gfortran libpng-dev freetype* libblas-dev liblapack-dev libatlas-base-dev python-qt4 python-tk libssl-dev libhdf5-dev
             $ sudo pip install numpy
             $ git clone http://jro-dev.igp.gob.pe/rhodecode/schain/
             $ cd schain
             $ sudo pip install ./
             ```
             **Its recommended to install schain in a virtual environment**
             ```
             $ sudo pip install virtualenv
             $ virtualenv /path/to/virtual --system-site-packages
             $ source /path/to/virtual/bin/activate
             (virtual) $ cd schain
             (virtual) $ pip install ./
             ```
             ## First Script
             Read Spectra data (.pdata) - remove dc - plot spectra & RTI
             Import SCh and creating a project
             ```python
             #!/usr/bin/python
             from schainpy.controller import Project
             controller = Project()
             controller.setup(id = '100',
                              name='test',
                              description='Basic experiment')
             ```
             Adding read unit and operations
             ```python
             read_unit = controller.addReadUnit(datatype='Spectra',
                                                path='/path/to/pdata/',
                                                startDate='2014/01/31',
                                                endDate='2014/03/31',
                                                startTime='00:00:00',
                                                endTime='23:59:59',
                                                online=0,
                                                walk=0)
             proc_unit = controller.addProcUnit(datatype='Spectra',
                                                inputId=read_unit.getId())
             op = proc_unit.addOperation(name='selectChannels')
             op.addParameter(name='channelList', value='0,1', format='intlist')
             op = proc_unit.addOperation(name='selectHeights')
             op.addParameter(name='minHei', value='80', format='float')
             op.addParameter(name='maxHei', value='200', format='float')
             op = proc_unit.addOperation(name='removeDC')
             ```
             Plotting data & start project
             ```python
             op = proc_unit.addOperation(name='SpectraPlot', optype='other')
             op.addParameter(name='id', value='1', format='int')
             op.addParameter(name='wintitle', value='Spectra', format='str')
             op = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
             op.addParameter(name='id', value='2', format='int')
             op.addParameter(name='wintitle', value='RTI', format='str')
             controller.start()
             ```
             Full script
             ```python
             #!/usr/bin/python
             from schainpy.controller import Project
             controller = Project()
             controller.setup(id = '100',
                              name='test',
                              description='Basic experiment')
             read_unit = controller.addReadUnit(datatype='Spectra',
                                                path='/path/to/pdata/',
                                                startDate='2014/01/31',
                                                endDate='2014/03/31',
                                                startTime='00:00:00',
                                                endTime='23:59:59',
                                                online=0,
                                                walk=0)
             proc_unit = controller.addProcUnit(datatype='Spectra',
                                                inputId=read_unit.getId())
             op = proc_unit.addOperation(name='selectChannels')
             op.addParameter(name='channelList', value='0,1', format='intlist')
             op = proc_unit.addOperation(name='selectHeights')
             op.addParameter(name='minHei', value='80', format='float')
             op.addParameter(name='maxHei', value='200', format='float')
             op = proc_unit.addOperation(name='removeDC')
             op = proc_unit.addOperation(name='SpectraPlot', optype='other')
             op.addParameter(name='id', value='6', format='int')
             op.addParameter(name='wintitle', value='Spectra', format='str')
             op = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
             op.addParameter(name='id', value='2', format='int')
             op.addParameter(name='wintitle', value='RTI', format='str')
             controller.start()
-            ```
+            ```
  No newline at end of file

schainpy/controller.py +40 -40

             '''
             Created on September , 2012
             @author:
             '''
             import sys
             import ast
             import datetime
             import traceback
             import math
             import time
             from multiprocessing import Process, Queue, cpu_count
             import schainpy
             import schainpy.admin
             from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
             from xml.dom import minidom
             from schainpy.model import *
             from time import sleep
             def prettify(elem):
                 """Return a pretty-printed XML string for the Element.
                 """
                 rough_string = tostring(elem, 'utf-8')
                 reparsed = minidom.parseString(rough_string)
                 return reparsed.toprettyxml(indent="  ")
             def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
                 skip = 0
                 cursor = 0
                 nFiles = None
                 processes = []
                 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
                 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
                 days = (dt2 - dt1).days
                 for day in range(days+1):
                     skip = 0
                     cursor = 0
                     q = Queue()
                     processes = []
                     dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
                     firstProcess = Process(target=child, args=(cursor, skip, q, dt))
                     firstProcess.start()
                     if by_day:
                         continue
                     nFiles = q.get()
                     if nFiles==0:
                         continue
                     firstProcess.terminate()
                     skip = int(math.ceil(nFiles/nProcess))
                     while True:
                         processes.append(Process(target=child, args=(cursor, skip, q, dt)))
                         processes[cursor].start()
                         if nFiles < cursor*skip:
                             break
                         cursor += 1
                     def beforeExit(exctype, value, trace):
                         for process in processes:
                             process.terminate()
                             process.join()
                         print traceback.print_tb(trace)
                     sys.excepthook = beforeExit
                     for process in processes:
                         process.join()
                         process.terminate()
                     time.sleep(3)
             class ParameterConf():
                 id = None
                 name = None
                 value = None
                 format = None
                 __formated_value = None
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.format = 'str'
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getValue(self):
                     value = self.value
                     format = self.format
                     if self.__formated_value != None:
                         return self.__formated_value
                     if format == 'obj':
                         return value
                     if format == 'str':
                         self.__formated_value = str(value)
                         return self.__formated_value
                     if value == '':
                         raise ValueError, "%s: This parameter value is empty" %self.name
                     if format == 'list':
                         strList = value.split(',')
                         self.__formated_value = strList
                         return self.__formated_value
                     if format == 'intlist':
                         """
                         Example:
                             value = (0,1,2)
                         """
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             new_value = [int(new_value)]
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'floatlist':
                         """
                         Example:
                             value = (0.5, 1.4, 2.7)
                         """
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             new_value = [float(new_value)]
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'date':
                         strList = value.split('/')
                         intList = [int(x) for x in strList]
                         date = datetime.date(intList[0], intList[1], intList[2])
                         self.__formated_value = date
                         return self.__formated_value
                     if format == 'time':
                         strList = value.split(':')
                         intList = [int(x) for x in strList]
                         time = datetime.time(intList[0], intList[1], intList[2])
                         self.__formated_value = time
                         return self.__formated_value
                     if format == 'pairslist':
                         """
                         Example:
                             value = (0,1),(1,2)
                         """
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             raise ValueError, "%s has to be a tuple or list of pairs" %value
                         if type(new_value[0]) not in (tuple, list):
                             if len(new_value) != 2:
                                 raise ValueError, "%s has to be a tuple or list of pairs" %value
                             new_value = [new_value]
                         for thisPair in new_value:
                             if len(thisPair) != 2:
                                 raise ValueError, "%s has to be a tuple or list of pairs" %value
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'multilist':
                         """
                         Example:
                             value = (0,1,2),(3,4,5)
                         """
                         multiList = ast.literal_eval(value)
                         if type(multiList[0]) == int:
                             multiList = ast.literal_eval("(" + value + ")")
                         self.__formated_value = multiList
                         return self.__formated_value
                     if format == 'bool':
                         value = int(value)
                     if format == 'int':
                         value = float(value)
                     format_func = eval(format)
                     self.__formated_value = format_func(value)
                     return self.__formated_value
                 def updateId(self, new_id):
                     self.id = str(new_id)
                 def setup(self, id, name, value, format='str'):
                     self.id = str(id)
                     self.name = name
                     if format == 'obj':
                         self.value = value
                     else:
                         self.value = str(value)
                     self.format = str.lower(format)
                     self.getValue()
                     return 1
                 def update(self, name, value, format='str'):
                     self.name = name
                     self.value = str(value)
                     self.format = format
                 def makeXml(self, opElement):
                     if self.name not in ('queue',):
                         parmElement = SubElement(opElement, self.ELEMENTNAME)
                         parmElement.set('id', str(self.id))
                         parmElement.set('name', self.name)
                         parmElement.set('value', self.value)
                         parmElement.set('format', self.format)
                 def readXml(self, parmElement):
                     self.id = parmElement.get('id')
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                     self.format = str.lower(parmElement.get('format'))
                     #Compatible with old signal chain version
                     if self.format == 'int' and self.name == 'idfigure':
                         self.name = 'id'
                 def printattr(self):
                     print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
             class OperationConf():
                 id = None
                 name = None
                 priority = None
                 type = None
                 parmConfObjList = []
                 ELEMENTNAME = 'Operation'
                 def __init__(self):
                     self.id = '0'
                     self.name = None
                     self.priority = None
                     self.type = 'self'
                 def __getNewId(self):
                     return int(self.id)*10 + len(self.parmConfObjList) + 1
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     n = 1
                     for parmObj in self.parmConfObjList:
                         idParm = str(int(new_id)*10 + n)
                         parmObj.updateId(idParm)
                         n += 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getParameterObjList(self):
                     return self.parmConfObjList
                 def getParameterObj(self, parameterName):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.name != parameterName:
                             continue
                         return parmConfObj
                     return None
                 def getParameterObjfromValue(self, parameterValue):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.getValue() != parameterValue:
                             continue
                         return parmConfObj.getValue()
                     return None
                 def getParameterValue(self, parameterName):
                     parameterObj = self.getParameterObj(parameterName)
-            #         if not parameterObj:
+                #         if not parameterObj:
-            #             return None
+                #             return None
                     value = parameterObj.getValue()
                     return value
                 def getKwargs(self):
                     kwargs = {}
                     for parmConfObj in self.parmConfObjList:
                         if self.name == 'run' and parmConfObj.name == 'datatype':
                             continue
                         kwargs[parmConfObj.name] = parmConfObj.getValue()
                     return kwargs
                 def setup(self, id, name, priority, type):
                     self.id = str(id)
                     self.name = name
                     self.type = type
                     self.priority = priority
                     self.parmConfObjList = []
                 def removeParameters(self):
                     for obj in self.parmConfObjList:
                         del obj
                     self.parmConfObjList = []
                 def addParameter(self, name, value, format='str'):
                     id = self.__getNewId()
                     parmConfObj = ParameterConf()
                     if not parmConfObj.setup(id, name, value, format):
                         return None
                     self.parmConfObjList.append(parmConfObj)
                     return parmConfObj
                 def changeParameter(self, name, value, format='str'):
                     parmConfObj = self.getParameterObj(name)
                     parmConfObj.update(name, value, format)
                     return parmConfObj
                 def makeXml(self, procUnitElement):
                     opElement = SubElement(procUnitElement, self.ELEMENTNAME)
                     opElement.set('id', str(self.id))
                     opElement.set('name', self.name)
                     opElement.set('type', self.type)
                     opElement.set('priority', str(self.priority))
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.makeXml(opElement)
                 def readXml(self, opElement):
                     self.id = opElement.get('id')
                     self.name = opElement.get('name')
                     self.type = opElement.get('type')
                     self.priority = opElement.get('priority')
                     #Compatible with old signal chain version
                     #Use of 'run' method instead 'init'
                     if self.type == 'self' and self.name == 'init':
                         self.name = 'run'
                     self.parmConfObjList = []
                     parmElementList = opElement.iter(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         #Compatible with old signal chain version
                         #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
                         if self.type != 'self' and self.name == 'Plot':
                             if parmConfObj.format == 'str' and parmConfObj.name == 'type':
                                 self.name = parmConfObj.value
                                 continue
                         self.parmConfObjList.append(parmConfObj)
                 def printattr(self):
                     print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
                                                                           self.id,
                                                                           self.name,
                                                                           self.type,
                                                                           self.priority)
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.printattr()
                 def createObject(self, plotter_queue=None):
                     if self.type == 'self':
                         raise ValueError, "This operation type cannot be created"
                     if self.type == 'plotter':
                         #Plotter(plotter_name)
                         if not plotter_queue:
                             raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
                         opObj = Plotter(self.name, plotter_queue)
                     if self.type == 'external' or self.type == 'other':
                         className = eval(self.name)
                         kwargs = self.getKwargs()
                         opObj = className(**kwargs)
                     return opObj
             class ProcUnitConf():
                 id = None
                 name = None
                 datatype = None
                 inputId = None
                 parentId = None
                 opConfObjList = []
                 procUnitObj = None
                 opObjList = []
                 ELEMENTNAME = 'ProcUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.opConfObjList = []
                     self.procUnitObj = None
                     self.opObjDict = {}
                 def __getPriority(self):
                     return len(self.opConfObjList)+1
                 def __getNewId(self):
                     return int(self.id)*10 + len(self.opConfObjList) + 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def updateId(self, new_id, parentId=parentId):
                     new_id = int(parentId)*10 + (int(self.id) % 10)
                     new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
                     #If this proc unit has not inputs
                     if self.inputId == '0':
                         new_inputId = 0
                     n = 1
                     for opConfObj in self.opConfObjList:
                         idOp = str(int(new_id)*10 + n)
                         opConfObj.updateId(idOp)
                         n += 1
                     self.parentId = str(parentId)
                     self.id = str(new_id)
                     self.inputId = str(new_inputId)
                 def getInputId(self):
                     return self.inputId
                 def getOperationObjList(self):
                     return self.opConfObjList
                 def getOperationObj(self, name=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.name != name:
                             continue
                         return opConfObj
                     return None
                 def getOpObjfromParamValue(self, value=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
                             continue
                         return opConfObj
                     return None
                 def getProcUnitObj(self):
                     return self.procUnitObj
                 def setup(self, id, name, datatype, inputId, parentId=None):
                     #Compatible with old signal chain version
                     if datatype==None and name==None:
                         raise ValueError, "datatype or name should be defined"
                     if name==None:
                         if 'Proc' in datatype:
                             name = datatype
                         else:
                             name = '%sProc' %(datatype)
                     if datatype==None:
                         datatype = name.replace('Proc','')
                     self.id = str(id)
                     self.name = name
                     self.datatype = datatype
                     self.inputId = inputId
                     self.parentId = parentId
                     self.opConfObjList = []
                     self.addOperation(name='run', optype='self')
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                     self.addOperation(name='run')
                 def addParameter(self, **kwargs):
                     '''
                     Add parameters to "run" operation
                     '''
                     opObj = self.opConfObjList[0]
                     opObj.addParameter(**kwargs)
                     return opObj
                 def addOperation(self, name, optype='self'):
                     id = self.__getNewId()
                     priority = self.__getPriority()
                     opConfObj = OperationConf()
                     opConfObj.setup(id, name=name, priority=priority, type=optype)
                     self.opConfObjList.append(opConfObj)
                     return opConfObj
                 def makeXml(self, projectElement):
                     procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
                     procUnitElement.set('id', str(self.id))
                     procUnitElement.set('name', self.name)
                     procUnitElement.set('datatype', self.datatype)
                     procUnitElement.set('inputId', str(self.inputId))
                     for opConfObj in self.opConfObjList:
                         opConfObj.makeXml(procUnitElement)
                 def readXml(self, upElement):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.inputId = upElement.get('inputId')
                     if self.ELEMENTNAME == "ReadUnit":
                         self.datatype = self.datatype.replace("Reader", "")
                     if self.ELEMENTNAME == "ProcUnit":
                         self.datatype = self.datatype.replace("Proc", "")
                     if self.inputId == 'None':
                         self.inputId = '0'
                     self.opConfObjList = []
                     opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement)
                         self.opConfObjList.append(opConfObj)
                 def printattr(self):
                     print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
-                                                                         self.id,
+                                                                            self.id,
-                                                                         self.name,
+                                                                            self.name,
-                                                                         self.datatype,
+                                                                            self.datatype,
-                                                                         self.inputId)
+                                                                            self.inputId)
                     for opConfObj in self.opConfObjList:
                         opConfObj.printattr()
                 def getKwargs(self):
                     opObj = self.opConfObjList[0]
                     kwargs = opObj.getKwargs()
                     return kwargs
                 def createObjects(self, plotter_queue=None):
                     className = eval(self.name)
                     kwargs = self.getKwargs()
                     procUnitObj = className(**kwargs)
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type=='self' and self.name=='run':
                             continue
                         elif opConfObj.type=='self':
                             procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
                             continue
                         opObj = opConfObj.createObject(plotter_queue)
                         self.opObjDict[opConfObj.id] = opObj
                         procUnitObj.addOperation(opObj, opConfObj.id)
                     self.procUnitObj = procUnitObj
                     return procUnitObj
                 def run(self):
                     is_ok = False
                     for opConfObj in self.opConfObjList:
                         kwargs = {}
                         for parmConfObj in opConfObj.getParameterObjList():
                             if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
                                 continue
                             kwargs[parmConfObj.name] = parmConfObj.getValue()
                         #ini = time.time()
                         #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
                         sts = self.procUnitObj.call(opType = opConfObj.type,
                                                     opName = opConfObj.name,
                                                     opId = opConfObj.id,
+                                                    **kwargs)
-            #             total_time = time.time() - ini
+                        #             total_time = time.time() - ini
+                        #
-            #             if total_time > 0.002:
+                        #             if total_time > 0.002:
-            #                 print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
+                        #                 print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
                         is_ok = is_ok or sts
                     return is_ok
                 def close(self):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self':
                             continue
                         opObj = self.procUnitObj.getOperationObj(opConfObj.id)
                         opObj.close()
                     self.procUnitObj.close()
                     return
             class ReadUnitConf(ProcUnitConf):
                 path = None
                 startDate = None
                 endDate = None
                 startTime = None
                 endTime = None
                 ELEMENTNAME = 'ReadUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.parentId = None
                     self.opConfObjList = []
                     self.opObjList = []
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
                           endTime="", parentId=None, queue=None, server=None, **kwargs):
                     #Compatible with old signal chain version
                     if datatype==None and name==None:
                         raise ValueError, "datatype or name should be defined"
                     if name==None:
                         if 'Reader' in datatype:
                             name = datatype
                         else:
                             name = '%sReader' %(datatype)
                     if datatype==None:
                         datatype = name.replace('Reader','')
                     self.id = id
                     self.name = name
                     self.datatype = datatype
                     if path != '':
                         self.path = os.path.abspath(path)
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.inputId = '0'
                     self.parentId = parentId
                     self.queue = queue
                     self.server = server
                     self.addRunOperation(**kwargs)
                 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
                     #Compatible with old signal chain version
                     if datatype==None and name==None:
                         raise ValueError, "datatype or name should be defined"
                     if name==None:
                         if 'Reader' in datatype:
                             name = datatype
                         else:
                             name = '%sReader' %(datatype)
                     if datatype==None:
                         datatype = name.replace('Reader','')
                     self.datatype = datatype
                     self.name = name
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.inputId = '0'
                     self.parentId = parentId
                     self.updateRunOperation(**kwargs)
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                 def addRunOperation(self, **kwargs):
                     opObj = self.addOperation(name = 'run', optype = 'self')
                     if self.server is None:
                         opObj.addParameter(name='datatype' , value=self.datatype, format='str')
                         opObj.addParameter(name='path'     , value=self.path, format='str')
                         opObj.addParameter(name='startDate' , value=self.startDate, format='date')
                         opObj.addParameter(name='endDate'   , value=self.endDate, format='date')
                         opObj.addParameter(name='startTime' , value=self.startTime, format='time')
                         opObj.addParameter(name='endTime'   , value=self.endTime, format='time')
                         opObj.addParameter(name='queue'   , value=self.queue, format='obj')
                         for key, value in kwargs.items():
                             opObj.addParameter(name=key, value=value, format=type(value).__name__)
                     else:
                         opObj.addParameter(name='server' , value=self.server, format='str')
                     return opObj
                 def updateRunOperation(self, **kwargs):
                     opObj = self.getOperationObj(name = 'run')
                     opObj.removeParameters()
                     opObj.addParameter(name='datatype' , value=self.datatype, format='str')
                     opObj.addParameter(name='path'     , value=self.path, format='str')
                     opObj.addParameter(name='startDate' , value=self.startDate, format='date')
                     opObj.addParameter(name='endDate'   , value=self.endDate, format='date')
                     opObj.addParameter(name='startTime' , value=self.startTime, format='time')
                     opObj.addParameter(name='endTime'   , value=self.endTime, format='time')
                     for key, value in kwargs.items():
                         opObj.addParameter(name=key, value=value, format=type(value).__name__)
                     return opObj
-            #     def makeXml(self, projectElement):
+                    #     def makeXml(self, projectElement):
+                    #
-            #         procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
+                    #         procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
-            #         procUnitElement.set('id', str(self.id))
+                    #         procUnitElement.set('id', str(self.id))
-            #         procUnitElement.set('name', self.name)
+                    #         procUnitElement.set('name', self.name)
-            #         procUnitElement.set('datatype', self.datatype)
+                    #         procUnitElement.set('datatype', self.datatype)
-            #         procUnitElement.set('inputId', str(self.inputId))
+                    #         procUnitElement.set('inputId', str(self.inputId))
+                    #
-            #         for opConfObj in self.opConfObjList:
+                    #         for opConfObj in self.opConfObjList:
-            #             opConfObj.makeXml(procUnitElement)
+                    #             opConfObj.makeXml(procUnitElement)
                 def readXml(self, upElement):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.inputId = upElement.get('inputId')
                     if self.ELEMENTNAME == "ReadUnit":
                         self.datatype = self.datatype.replace("Reader", "")
                     if self.inputId == 'None':
                         self.inputId = '0'
                     self.opConfObjList = []
                     opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement)
                         self.opConfObjList.append(opConfObj)
                         if opConfObj.name == 'run':
                             self.path = opConfObj.getParameterValue('path')
                             self.startDate = opConfObj.getParameterValue('startDate')
                             self.endDate = opConfObj.getParameterValue('endDate')
                             self.startTime = opConfObj.getParameterValue('startTime')
                             self.endTime = opConfObj.getParameterValue('endTime')
             class Project():
                 id = None
                 name = None
                 description = None
                 filename = None
                 procUnitConfObjDict = None
                 ELEMENTNAME = 'Project'
                 plotterQueue = None
                 def __init__(self, plotter_queue=None):
                     self.id = None
                     self.name = None
                     self.description = None
                     self.plotterQueue = plotter_queue
                     self.procUnitConfObjDict = {}
                 def __getNewId(self):
                     idList = self.procUnitConfObjDict.keys()
                     id = int(self.id)*10
                     while True:
                         id += 1
                         if str(id) in idList:
                             continue
                         break
                     return str(id)
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     keyList = self.procUnitConfObjDict.keys()
                     keyList.sort()
                     n = 1
                     newProcUnitConfObjDict = {}
                     for procKey in keyList:
                         procUnitConfObj = self.procUnitConfObjDict[procKey]
                         idProcUnit = str(int(self.id)*10 + n)
                         procUnitConfObj.updateId(idProcUnit, parentId = self.id)
                         newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
                         n += 1
                     self.procUnitConfObjDict = newProcUnitConfObjDict
                 def setup(self, id, name, description):
                     self.id = str(id)
                     self.name = name
                     self.description = description
                 def update(self, name, description):
                     self.name = name
                     self.description = description
                 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
                     if id is None:
                         idReadUnit = self.__getNewId()
                     else:
                         idReadUnit = str(id)
                     readUnitConfObj = ReadUnitConf()
                     readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
                     self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     return readUnitConfObj
                 def addProcUnit(self, inputId='0', datatype=None, name=None):
                     idProcUnit = self.__getNewId()
                     procUnitConfObj = ProcUnitConf()
                     procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
                     self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     return procUnitConfObj
                 def removeProcUnit(self, id):
                     if id in self.procUnitConfObjDict.keys():
                         self.procUnitConfObjDict.pop(id)
                 def getReadUnitId(self):
                     readUnitConfObj = self.getReadUnitObj()
                     return readUnitConfObj.id
                 def getReadUnitObj(self):
                     for obj in self.procUnitConfObjDict.values():
                         if obj.getElementName() == "ReadUnit":
                             return obj
                     return None
                 def getProcUnitObj(self, id=None, name=None):
                     if id != None:
                         return self.procUnitConfObjDict[id]
                     if name != None:
                         return self.getProcUnitObjByName(name)
                     return None
                 def getProcUnitObjByName(self, name):
                     for obj in self.procUnitConfObjDict.values():
                         if obj.name == name:
                             return obj
                     return None
                 def procUnitItems(self):
                     return self.procUnitConfObjDict.items()
                 def makeXml(self):
                     projectElement = Element('Project')
                     projectElement.set('id', str(self.id))
                     projectElement.set('name', self.name)
                     projectElement.set('description', self.description)
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.makeXml(projectElement)
                     self.projectElement = projectElement
                 def writeXml(self, filename=None):
                     if filename == None:
                         if self.filename:
                             filename = self.filename
                         else:
                             filename = "schain.xml"
                     if not filename:
                         print "filename has not been defined. Use setFilename(filename) for do it."
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.access(os.path.dirname(abs_file), os.W_OK):
                         print "No write permission on %s" %os.path.dirname(abs_file)
                         return 0
                     if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
                         print "File %s already exists and it could not be overwriten" %abs_file
                         return 0
                     self.makeXml()
                     ElementTree(self.projectElement).write(abs_file, method='xml')
                     self.filename = abs_file
                     return 1
                 def readXml(self, filename = None):
                     if not filename:
                         print "filename is not defined"
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.path.isfile(abs_file):
                         print "%s file does not exist" %abs_file
                         return 0
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     try:
                         self.projectElement = ElementTree().parse(abs_file)
                     except:
                         print "Error reading %s, verify file format" %filename
                         return 0
                     self.project = self.projectElement.tag
                     self.id = self.projectElement.get('id')
                     self.name = self.projectElement.get('name')
                     self.description = self.projectElement.get('description')
                     readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
                     for readUnitElement in readUnitElementList:
                         readUnitConfObj = ReadUnitConf()
                         readUnitConfObj.readXml(readUnitElement)
                         if readUnitConfObj.parentId == None:
                             readUnitConfObj.parentId = self.id
                         self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
                     for procUnitElement in procUnitElementList:
                         procUnitConfObj = ProcUnitConf()
                         procUnitConfObj.readXml(procUnitElement)
                         if procUnitConfObj.parentId == None:
                             procUnitConfObj.parentId = self.id
                         self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     self.filename = abs_file
                     return 1
                 def printattr(self):
                     print "Project[%s]: name = %s, description = %s" %(self.id,
-                                                                          self.name,
+                                                                    self.name,
-                                                                          self.description)
+                                                                    self.description)
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.printattr()
                 def createObjects(self):
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.createObjects(self.plotterQueue)
                 def __connect(self, objIN, thisObj):
                     thisObj.setInput(objIN.getOutputObj())
                 def connectObjects(self):
                     for thisPUConfObj in self.procUnitConfObjDict.values():
                         inputId = thisPUConfObj.getInputId()
                         if int(inputId) == 0:
                             continue
                         #Get input object
                         puConfINObj = self.procUnitConfObjDict[inputId]
                         puObjIN = puConfINObj.getProcUnitObj()
                         #Get current object
                         thisPUObj = thisPUConfObj.getProcUnitObj()
                         self.__connect(puObjIN, thisPUObj)
                 def __handleError(self, procUnitConfObj, send_email=True):
                     import socket
                     err = traceback.format_exception(sys.exc_info()[0],
-                                                     sys.exc_info()[1],
+                                                    sys.exc_info()[1],
-                                                     sys.exc_info()[2])
+                                                    sys.exc_info()[2])
                     print "***** Error occurred in %s *****" %(procUnitConfObj.name)
                     print "***** %s" %err[-1]
                     message = "".join(err)
                     sys.stderr.write(message)
                     if not send_email:
                         return
                     subject =  "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
                     subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
                     subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
                     subtitle += "Working directory: %s\n" %os.path.abspath("./")
                     subtitle += "Configuration file: %s\n" %self.filename
                     subtitle += "Time: %s\n" %str(datetime.datetime.now())
                     readUnitConfObj = self.getReadUnitObj()
                     if readUnitConfObj:
                         subtitle += "\nInput parameters:\n"
                         subtitle += "[Data path = %s]\n" %readUnitConfObj.path
                         subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
                         subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
                         subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
                         subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
                         subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
                     adminObj = schainpy.admin.SchainNotify()
                     adminObj.sendAlert(message=message,
-                                       subject=subject,
+                                    subject=subject,
-                                       subtitle=subtitle,
+                                    subtitle=subtitle,
-                                       filename=self.filename)
+                                    filename=self.filename)
                 def isPaused(self):
                     return 0
                 def isStopped(self):
                     return 0
                 def runController(self):
                     """
                     returns 0 when this process has been stopped, 1 otherwise
                     """
                     if self.isPaused():
                         print "Process suspended"
                         while True:
                             sleep(0.1)
                             if not self.isPaused():
                                 break
                             if self.isStopped():
                                 break
                         print "Process reinitialized"
                     if self.isStopped():
                         print "Process stopped"
                         return 0
                     return 1
                 def setFilename(self, filename):
                     self.filename = filename
                 def setPlotterQueue(self, plotter_queue):
                     raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
                 def getPlotterQueue(self):
                     raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
                 def useExternalPlotter(self):
                     raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
                 def run(self):
                     print
                     print "*"*60
                     print "   Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
                     print "*"*60
                     print
                     keyList = self.procUnitConfObjDict.keys()
                     keyList.sort()
                     while(True):
                         is_ok = False
                         for procKey in keyList:
             #                 print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
                             procUnitConfObj = self.procUnitConfObjDict[procKey]
                             try:
                                 sts = procUnitConfObj.run()
                                 is_ok = is_ok or sts
                             except KeyboardInterrupt:
                                 is_ok = False
                                 break
                             except ValueError, e:
                                 sleep(0.5)
                                 self.__handleError(procUnitConfObj, send_email=True)
                                 is_ok = False
                                 break
                             except:
                                 sleep(0.5)
                                 self.__handleError(procUnitConfObj)
                                 is_ok = False
                                 break
                         #If every process unit finished so end process
                         if not(is_ok):
             #                 print "Every process unit have finished"
                             break
                         if not self.runController():
                             break
                     #Closing every process
                     for procKey in keyList:
                         procUnitConfObj = self.procUnitConfObjDict[procKey]
                         procUnitConfObj.close()
                     print "Process finished"
                 def start(self, filename=None):
                     self.writeXml(filename)
                     self.createObjects()
                     self.connectObjects()
                     self.run()

schainpy/controller_api.py +1 -1

             import threading
             from Queue import Queue
             from schainpy.controller import Project
             from schainpy.model.graphics.jroplotter import PlotManager
             class ControllerThread(threading.Thread, Project):
                 def __init__(self, plotter_queue=None):
                     threading.Thread.__init__(self)
                     Project.__init__(self, plotter_queue)
                     self.setDaemon(True)
                     self.lock = threading.Lock()
                     self.control = {'stop':False, 'pause':False}
                 def __del__(self):
                     self.control['stop'] = True
                 def stop(self):
                     self.lock.acquire()
                     self.control['stop'] = True
                     self.lock.release()
                 def pause(self):
                     self.lock.acquire()
                     self.control['pause'] = not(self.control['pause'])
                     paused = self.control['pause']
                     self.lock.release()
                     return paused
                 def isPaused(self):
                     self.lock.acquire()
                     paused = self.control['pause']
                     self.lock.release()
                     return paused
                 def isStopped(self):
                     self.lock.acquire()
                     stopped = self.control['stop']
                     self.lock.release()
                     return stopped
                 def run(self):
                     self.control['stop'] = False
                     self.control['pause'] = False
                     self.writeXml()
                     self.createObjects()
                     self.connectObjects()
                     Project.run(self)
                 def isRunning(self):
                     return self.is_alive()
                 def isFinished(self):
                     return not self.is_alive()
                 def setPlotters(self):
                     plotterList = PlotManager.plotterList
                     for thisPUConfObj in self.procUnitConfObjDict.values():
                         inputId = thisPUConfObj.getInputId()
                         if int(inputId) == 0:
                             continue
                         for thisOpObj in thisPUConfObj.getOperationObjList():
                             if thisOpObj.type == "self":
                                 continue
                             if thisOpObj.name in plotterList:
-                                thisOpObj.type = "plotter"
+                                thisOpObj.type = "other"
                 def setPlotterQueue(self, plotter_queue):
                     self.plotterQueue = plotter_queue
                 def getPlotterQueue(self):
                     return self.plotterQueue
                 def useExternalPlotter(self):
                     self.plotterQueue = Queue(10)
                     self.setPlotters()
                     plotManagerObj = PlotManager(self.plotterQueue)
                     plotManagerObj.setController(self)
                     return plotManagerObj
             # from PyQt4 import QtCore
             # from PyQt4.QtCore import SIGNAL
             #
             # class ControllerQThread(QtCore.QThread, Project):
             #
             #     def __init__(self, filename):
             #
             #         QtCore.QThread.__init__(self)
             #         Project.__init__(self)
             #
             #         self.filename = filename
             #
             #         self.lock = threading.Lock()
             #         self.control = {'stop':False, 'pause':False}
             #
             #     def __del__(self):
             #
             #         self.control['stop'] = True
             #         self.wait()
             #
             #     def stop(self):
             #
             #         self.lock.acquire()
             #
             #         self.control['stop'] = True
             #
             #         self.lock.release()
             #
             #     def pause(self):
             #
             #         self.lock.acquire()
             #
             #         self.control['pause'] = not(self.control['pause'])
             #         paused = self.control['pause']
             #
             #         self.lock.release()
             #
             #         return paused
             #
             #     def isPaused(self):
             #
             #         self.lock.acquire()
             #         paused = self.control['pause']
             #         self.lock.release()
             #
             #         return paused
             #
             #     def isStopped(self):
             #
             #         self.lock.acquire()
             #         stopped = self.control['stop']
             #         self.lock.release()
             #
             #         return stopped
             #
             #     def run(self):
             #
             #         self.control['stop'] = False
             #         self.control['pause'] = False
             #
             #         self.readXml(self.filename)
             #         self.createObjects()
             #         self.connectObjects()
             #         self.emit( SIGNAL( "jobStarted( PyQt_PyObject )" ), 1)
             #         Project.run(self)
             #         self.emit( SIGNAL( "jobFinished( PyQt_PyObject )" ), 1)
             #

schainpy/model/utils/jroutils_ftp.py +5 -3

             '''
             @author: Daniel Suarez
             '''
             import os
             import glob
             import ftplib
             try:
                 import paramiko
                 import scp
             except:
                 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
             import time
             import threading
             Thread = threading.Thread
             # try:
             #     from gevent import sleep
             # except:
             from time import sleep
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             class Remote(Thread):
                 """
                 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
                 used to upload or download files remotely.
                 Non-standard Python modules used:
                     None
                 Written by:
                     "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe  Jun. 03, 2015
                 """
                 server = None
                 username = None
                 password = None
                 remotefolder = None
                 period = 60
                 fileList = []
                 bussy = False
                 def __init__(self, server, username, password, remotefolder, period=60):
                     Thread.__init__(self)
                     self.setDaemon(True)
                     self.status = 0
                     self.__server = server
                     self.__username = username
                     self.__password = password
                     self.__remotefolder = remotefolder
                     self.period = period
                     self.fileList = []
                     self.bussy = False
                     self.stopFlag = False
                     print "[Remote Server] Opening server: %s" %self.__server
                     if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
                         print "[Remote Server] %s server was opened successfully" %self.__server
                     self.close()
                     self.mutex = threading.Lock()
                 def stop(self):
                     self.stopFlag = True
                     self.join(10)
                 def open(self):
                     """
                     Connect to server and create a connection class (FTP or SSH) to remote server.
                     """
                     raise NotImplementedError, "Implement this method in child class"
                 def close(self):
                     """
                     Close connection to server
                     """
                     raise NotImplementedError, "Implement this method in child class"
                 def mkdir(self, remotefolder):
                     """
                     Create a folder remotely
                     """
                     raise NotImplementedError, "Implement this method in child class"
                 def cd(self, remotefolder):
                     """
                     Change working directory in remote server
                     """
                     raise NotImplementedError, "Implement this method in child class"
                 def download(self, filename, localfolder=None):
                     """
                     Download a file from server to local host
                     """
                     raise NotImplementedError, "Implement this method in child class"
                 def sendFile(self, fullfilename):
                     """
                     sendFile method is used to upload a local file to the current directory in remote server
                     Inputs:
                         fullfilename        - full path name of local file to store in remote directory
                     Returns:
 in error case else 1
                     """
                     raise NotImplementedError, "Implement this method in child class"
                 def upload(self, fullfilename, remotefolder=None):
                     """
                     upload method is used to upload a local file to remote directory. This method changes
                     working directory before sending a file.
                     Inputs:
                         fullfilename        - full path name of local file to store in remote directory
                         remotefolder    - remote directory
                     Returns:
 in error case else 1
                     """
                     print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
                     if not self.status:
                         return 0
                     if remotefolder == None:
                         remotefolder = self.remotefolder
                     if not self.cd(remotefolder):
                         return 0
                     if not self.sendFile(fullfilename):
                         print "[Remote Server] Error uploading file %s" %fullfilename
                         return 0
                     print "[Remote Server] upload finished successfully"
                     return 1
                 def delete(self, filename):
                     """
                     Remove a file from remote server
                     """
                     pass
                 def updateFileList(self, fileList):
                     """
                     Remove a file from remote server
                     """
                     if fileList == self.fileList:
                         return 0
                     self.mutex.acquire()
             #         init = time.time()
             #
             #         while(self.bussy):
             #             sleep(0.1)
             #             if time.time() - init > 2*self.period:
             #                 return 0
                     self.fileList = fileList
                     self.mutex.release()
                     return 1
                 def run(self):
                     if not self.status:
                         print "Finishing FTP service"
                         return
                     if not self.cd(self.remotefolder):
                         raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
                     while True:
                         for i in range(self.period):
                             if self.stopFlag:
                                 break
                             sleep(1)
                         if self.stopFlag:
                             break
             #             self.bussy = True
                         self.mutex.acquire()
                         print "[Remote Server] Opening %s" %self.__server
                         if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
                             self.mutex.release()
                             continue
                         for thisFile in self.fileList:
                             self.upload(thisFile, self.remotefolder)
                         print "[Remote Server] Closing %s" %self.__server
                         self.close()
                         self.mutex.release()
             #             self.bussy = False
                     print "[Remote Server] Thread stopped successfully"
             class FTPClient(Remote):
                 __ftpClientObj = None
                 def __init__(self, server, username, password, remotefolder, period=60):
                     """
                     """
                     Remote.__init__(self, server, username, password, remotefolder, period)
                 def open(self, server, username, password, remotefolder):
                     """
                     This method is used to set FTP parameters and establish a connection to remote server
                     Inputs:
                         server    - remote server IP Address
                         username    - remote server Username
                         password    - remote server password
                         remotefolder    - remote server current working directory
                     Return:
                         Boolean     -     Returns 1 if a connection has been established, 0 otherwise
                     Affects:
                         self.status        - in case of error or fail connection this parameter is set to 0 else 1
                     """
                     if server == None:
                         raise ValueError, "FTP server should be defined"
                     if username == None:
                         raise ValueError, "FTP username should be defined"
                     if password == None:
                         raise ValueError, "FTP password should be defined"
                     if remotefolder == None:
                         raise ValueError, "FTP remote folder should be defined"
                     try:
                         ftpClientObj = ftplib.FTP(server)
                     except ftplib.all_errors, e:
                         print "[FTP Server]: FTP server connection fail: %s" %server
                         print "[FTP Server]:", e
                         self.status = 0
                         return 0
                     try:
                         ftpClientObj.login(username, password)
                     except ftplib.all_errors:
                         print "[FTP Server]: FTP username or password are incorrect"
                         self.status = 0
                         return 0
                     if remotefolder == None:
                         remotefolder = ftpClientObj.pwd()
                     else:
                         try:
                             ftpClientObj.cwd(remotefolder)
                         except ftplib.all_errors:
                             print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
                             remotefolder = ftpClientObj.pwd()
                     self.server = server
                     self.username = username
                     self.password = password
                     self.remotefolder = remotefolder
                     self.__ftpClientObj = ftpClientObj
                     self.status = 1
                     return 1
                 def close(self):
                     """
                     Close connection to remote server
                     """
                     if not self.status:
                         return 0
                     self.__ftpClientObj.close()
                 def mkdir(self, remotefolder):
                     """
                     mkdir is used to make a new directory in remote server
                     Input:
                         remotefolder    - directory name
                     Return:
 in error case else 1
                     """
                     if not self.status:
                         return 0
                     try:
                         self.__ftpClientObj.mkd(dirname)
                     except ftplib.all_errors:
                         print "[FTP Server]: Error creating remote folder: %s" %remotefolder
                         return 0
                     return 1
                 def cd(self, remotefolder):
                     """
                     cd is used to change remote working directory on server
                     Input:
                         remotefolder    - current working directory
                     Affects:
                         self.remotefolder
                     Return:
 in case of error else 1
                     """
                     if not self.status:
                         return 0
                     if remotefolder == self.remotefolder:
                         return 1
                     try:
                         self.__ftpClientObj.cwd(remotefolder)
                     except ftplib.all_errors:
                         print '[FTP Server]: Error changing to %s' %remotefolder
                         print '[FTP Server]: Trying to create remote folder'
                         if not self.mkdir(remotefolder):
                             print '[FTP Server]: Remote folder could not be created'
                             return 0
                         try:
                             self.__ftpClientObj.cwd(remotefolder)
                         except ftplib.all_errors:
                             return 0
                     self.remotefolder = remotefolder
                     return 1
                 def sendFile(self, fullfilename):
                     if not self.status:
                         return 0
                     fp = open(fullfilename, 'rb')
                     filename = os.path.basename(fullfilename)
                     command = "STOR %s" %filename
                     try:
                         self.__ftpClientObj.storbinary(command, fp)
                     except ftplib.all_errors, e:
                         print "[FTP Server]:", e
                         return 0
                     try:
                         self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
                     except ftplib.all_errors, e:
                         print "[FTP Server]:", e
                     fp.close()
                     return 1
             class SSHClient(Remote):
                 __sshClientObj = None
                 __scpClientObj = None
                 def __init__(self, server, username, password, remotefolder, period=60):
                     """
                     """
                     Remote.__init__(self, server, username, password, remotefolder, period)
                 def open(self, server, username, password, remotefolder, port=22):
                     """
                     This method is used to set SSH parameters and establish a connection to a remote server
                     Inputs:
                         server    - remote server IP Address
                         username    - remote server Username
                         password    - remote server password
                         remotefolder    - remote server current working directory
                     Return: void
                     Affects:
                         self.status        - in case of error or fail connection this parameter is set to 0 else 1
                     """
                     import socket
                     if server == None:
                         raise ValueError, "SSH server should be defined"
                     if username == None:
                         raise ValueError, "SSH username should be defined"
                     if password == None:
                         raise ValueError, "SSH password should be defined"
                     if remotefolder == None:
                         raise ValueError, "SSH remote folder should be defined"
                     sshClientObj = paramiko.SSHClient()
                     sshClientObj.load_system_host_keys()
                     sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
                     self.status = 0
                     try:
                         sshClientObj.connect(server, username=username, password=password, port=port)
                     except paramiko.AuthenticationException, e:
             #             print "SSH username or password are incorrect: %s"
                         print "[SSH Server]:", e
                         return 0
                     except SSHException, e:
                         print "[SSH Server]:", e
                         return 0
                     except socket.error:
                         self.status = 0
                         print "[SSH Server]:", e
                         return 0
                     self.status = 1
                     scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
                     if remotefolder == None:
                         remotefolder = self.pwd()
                     self.server = server
                     self.username = username
                     self.password = password
                     self.__sshClientObj = sshClientObj
                     self.__scpClientObj = scpClientObj
                     self.status = 1
                     if not self.cd(remotefolder):
                         raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
                         return 0
                     self.remotefolder = remotefolder
                     return 1
                 def close(self):
                     """
                     Close connection to remote server
                     """
                     if not self.status:
                         return 0
                     self.__scpClientObj.close()
                     self.__sshClientObj.close()
                 def __execute(self, command):
                     """
                     __execute a command on remote server
                     Input:
                         command    - Exmaple 'ls -l'
                     Return:
 in error case else 1
                     """
                     if not self.status:
                         return 0
                     stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
                     result = stderr.readlines()
                     if len(result) > 1:
                         return 0
                     result = stdout.readlines()
                     if len(result) > 1:
                         return result[0][:-1]
                     return 1
                 def mkdir(self, remotefolder):
                     """
                     mkdir is used to make a new directory in remote server
                     Input:
                         remotefolder    - directory name
                     Return:
 in error case else 1
                     """
                     command = 'mkdir %s' %remotefolder
                     return self.__execute(command)
                 def pwd(self):
                     command = 'pwd'
                     return self.__execute(command)
                 def cd(self, remotefolder):
                     """
                     cd is used to change remote working directory on server
                     Input:
                         remotefolder    - current working directory
                     Affects:
                         self.remotefolder
                     Return:
 in case of error else 1
                     """
                     if not self.status:
                         return 0
                     if remotefolder == self.remotefolder:
                         return 1
                     chk_command = "cd %s; pwd" %remotefolder
                     mkdir_command = "mkdir %s" %remotefolder
                     if not self.__execute(chk_command):
                         if not self.__execute(mkdir_command):
                             self.remotefolder = None
                             return 0
                     self.remotefolder = remotefolder
                     return 1
                 def sendFile(self, fullfilename):
                     if not self.status:
                         return 0
                     try:
                         self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
                     except scp.ScpError, e:
                         print "[SSH Server]", str(e)
                         return 0
                     remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
                     command = 'chmod 775 %s' %remotefile
                     return self.__execute(command)
             class SendToServer(ProcessingUnit):
                 def __init__(self, **kwargs):
                     ProcessingUnit.__init__(self, **kwargs)
                     self.isConfig = False
                     self.clientObj = None
                 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
                     self.clientObj = None
                     self.localfolder = localfolder
                     self.ext = ext
                     self.period = period
                     if str.lower(protocol) == 'ftp':
                         self.clientObj = FTPClient(server, username, password, remotefolder, period)
                     if str.lower(protocol) == 'ssh':
                         self.clientObj = SSHClient(server, username, password, remotefolder, period)
                     if not self.clientObj:
                         raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
                     self.clientObj.start()
                 def findFiles(self):
                     if not type(self.localfolder) == list:
                         folderList = [self.localfolder]
                     else:
                         folderList = self.localfolder
                     #Remove duplicate items
                     folderList = list(set(folderList))
                     fullfilenameList = []
                     for thisFolder in folderList:
                         print "[Remote Server]: Searching files on %s" %thisFolder
                         filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
                         if len(filenameList) < 1:
                             continue
                         for thisFile in filenameList:
                             fullfilename = os.path.join(thisFolder, thisFile)
                             if fullfilename in fullfilenameList:
                                 continue
                             #Only files modified in the last 30 minutes are considered
                             if os.path.getmtime(fullfilename) < time.time() - 30*60:
                                 continue
                             fullfilenameList.append(fullfilename)
                     return fullfilenameList
                 def run(self, **kwargs):
                     if not self.isConfig:
                         self.init = time.time()
                         self.setup(**kwargs)
                         self.isConfig = True
+                    if not self.clientObj.is_alive():
+                        print "[Remote Server]: Restarting connection "
+                        self.setup(**kwargs)
                     if time.time() - self.init >= self.period:
                         fullfilenameList = self.findFiles()
                         if self.clientObj.updateFileList(fullfilenameList):
                             print "[Remote Server]: Sending the next files ", str(fullfilenameList)
                         self.init = time.time()
                 def close(self):
                     print "[Remote Server] Stopping thread"
                     self.clientObj.stop()
             class FTP(object):
                 """
                 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
                 Non-standard Python modules used: None
                 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe  Oct. 26, 2010
                 """
                 def __init__(self,server = None, username=None, password=None, remotefolder=None):
                     """
                     This method is used to setting parameters for FTP and establishing connection to remote server
                     Inputs:
                         server    - remote server IP Address
                         username    - remote server Username
                         password    - remote server password
                         remotefolder    - remote server current working directory
                     Return: void
                     Affects:
                         self.status    - in Error Case or Connection Failed this parameter is set to 1 else 0
                         self.folderList    - sub-folder list of remote folder
                         self.fileList    - file list of remote folder
                     """
                     if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
                         server, username, password, remotefolder = self.parmsByDefault()
                     self.server = server
                     self.username = username
                     self.password = password
                     self.remotefolder = remotefolder
                     self.file = None
                     self.ftp = None
                     self.status = 0
                     try:
                         self.ftp = ftplib.FTP(self.server)
                         self.ftp.login(self.username,self.password)
                         self.ftp.cwd(self.remotefolder)
             #            print 'Connect to FTP Server: Successfully'
                     except ftplib.all_errors:
                         print 'Error FTP Service'
                         self.status = 1
                         return
                     self.dirList = []
                     try:
                         self.dirList = self.ftp.nlst()
                     except ftplib.error_perm, resp:
                         if str(resp) == "550 No files found":
                                 print "no files in this directory"
                                 self.status = 1
                                 return
                     except ftplib.all_errors:
                         print 'Error Displaying Dir-Files'
                         self.status = 1
                         return
                     self.fileList = []
                     self.folderList = []
                     #only for test
                     for f in self.dirList:
                         name, ext = os.path.splitext(f)
                         if ext != '':
                             self.fileList.append(f)
             #                print 'filename: %s - size: %d'%(f,self.ftp.size(f))
                 def parmsByDefault(self):
                     server = 'jro-app.igp.gob.pe'
                     username = 'wmaster'
                     password = 'mst2010vhf'
                     remotefolder = '/home/wmaster/graficos'
                     return server, username, password, remotefolder
                 def mkd(self,dirname):
                     """
                     mkd is used to make directory in remote server
                     Input:
                         dirname    - directory name
                     Return:
 in error case else 0
                     """
                     try:
                         self.ftp.mkd(dirname)
                     except:
                         print 'Error creating remote folder:%s'%dirname
                         return 1
                     return 0
                 def delete(self,filename):
                     """
                     delete is used to delete file in current working directory of remote server
                     Input:
                         filename    - filename to delete in remote folder
                     Return:
 in error case else 0
                     """
                     try:
                         self.ftp.delete(filename)
                     except:
                         print 'Error deleting remote file:%s'%filename
                         return 1
                     return 0
                 def download(self,filename,localfolder):
                     """
                     download is used to downloading file from remote folder into local folder
                     Inputs:
                         filename    - filename to donwload
                         localfolder    - directory local to store filename
                     Returns:
                         self.status    - 1 in error case else 0
                     """
                     self.status = 0
                     if not(filename in self.fileList):
                         print 'filename:%s not exists'%filename
                         self.status = 1
                         return self.status
                     newfilename = os.path.join(localfolder,filename)
                     self.file = open(newfilename, 'wb')
                     try:
                         print 'Download: ' + filename
                         self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
                         print 'Download Complete'
                     except ftplib.all_errors:
                         print 'Error Downloading ' + filename
                         self.status = 1
                         return self.status
                     self.file.close()
                     return self.status
                 def __handleDownload(self,block):
                     """
                     __handleDownload is used to handle writing file
                     """
                     self.file.write(block)
                 def upload(self,filename,remotefolder=None):
                     """
                     upload is used to uploading local file to remote directory
                     Inputs:
                         filename    - full path name of local file to store in remote directory
                         remotefolder    - remote directory
                     Returns:
                         self.status    - 1 in error case else 0
                     """
                     if remotefolder == None:
                         remotefolder = self.remotefolder
                     self.status = 0
                     try:
                         self.ftp.cwd(remotefolder)
                         self.file = open(filename, 'rb')
                         (head, tail) = os.path.split(filename)
                         command = "STOR " + tail
                         print 'Uploading: ' + tail
                         self.ftp.storbinary(command, self.file)
                         print 'Upload Completed'
                     except ftplib.all_errors:
                         print 'Error Uploading ' + tail
                         self.status = 1
                         return self.status
                     self.file.close()
                     #back to initial directory in __init__()
                     self.ftp.cwd(self.remotefolder)
                     return self.status
                 def dir(self,remotefolder):
                     """
                     dir is used to change working directory of remote server and get folder and file list
                     Input:
                         remotefolder    - current working directory
                     Affects:
                         self.fileList    - file list of working directory
                     Return:
                         infoList    - list with filenames and size of file in bytes
                         self.folderList    -    folder list
                     """
                     self.remotefolder = remotefolder
                     print 'Change to ' + self.remotefolder
                     try:
                         self.ftp.cwd(remotefolder)
                     except ftplib.all_errors:
                         print 'Error Change to ' + self.remotefolder
                         infoList = None
                         self.folderList = None
                         return infoList,self.folderList
                     self.dirList = []
                     try:
                         self.dirList = self.ftp.nlst()
                     except ftplib.error_perm, resp:
                         if str(resp) == "550 No files found":
                                 print "no files in this directory"
                                 infoList = None
                                 self.folderList = None
                                 return infoList,self.folderList
                     except ftplib.all_errors:
                         print 'Error Displaying Dir-Files'
                         infoList = None
                         self.folderList = None
                         return infoList,self.folderList
                     infoList = []
                     self.fileList = []
                     self.folderList = []
                     for f in self.dirList:
                         name,ext = os.path.splitext(f)
                         if ext != '':
                             self.fileList.append(f)
                             value = (f,self.ftp.size(f))
                             infoList.append(value)
                         if ext == '':
                             self.folderList.append(f)
                     return infoList,self.folderList
                 def close(self):
                     """
                     close is used to close and end FTP connection
                     Inputs: None
                     Return: void
                     """
                     self.ftp.close()
             class SendByFTP(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.status = 1
                     self.counter = 0
                 def error_print(self, ValueError):
                     print ValueError, 'Error FTP'
                     print "don't worry the program is running..."
                 def worker_ftp(self, server, username, password, remotefolder, filenameList):
                     self.ftpClientObj = FTP(server, username, password, remotefolder)
                     for filename in filenameList:
                         self.ftpClientObj.upload(filename)
                     self.ftpClientObj.close()
                 def ftp_thread(self, server, username, password, remotefolder):
                     if not(self.status):
                         return
                     import multiprocessing
                     p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
                     p.start()
                     p.join(3)
                     if p.is_alive():
                         p.terminate()
                         p.join()
                         print 'killing ftp process...'
                         self.status = 0
                         return
                     self.status = 1
                     return
                 def filterByExt(self, ext, localfolder):
                     fnameList = glob.glob1(localfolder,ext)
                     self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
                     if len(self.filenameList) == 0:
                         self.status = 0
                 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
                     self.counter += 1
                     if self.counter >= period:
                         self.filterByExt(ext, localfolder)
                         self.ftp_thread(server, username, password, remotefolder)
                         self.counter = 0
                     self.status = 1

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages