schain Commit - r1011:6db081ca94fe · Jicamarca Repository

this fixes bug #1074 and bug #1101

José Chávez -

r1011:6db081ca94fe

parent child

Context file:

r1011:6db081ca94fe

Collapse all files

.vscode/c_cpp_properties.json created 644 +69 0

@@ -0,0 +1,69
	1	{
	2	"configurations": [
	3	{
	4	"name": "Mac",
	5	"includePath": [
	6	"/usr/include",
	7	"/usr/local/include",
	8	"${workspaceRoot}"
	9	],
	10	"defines": [],
	11	"intelliSenseMode": "clang-x64",
	12	"browse": {
	13	"path": [
	14	"/usr/include",
	15	"/usr/local/include",
	16	"${workspaceRoot}"
	17	],
	18	"limitSymbolsToIncludedHeaders": true,
	19	"databaseFilename": ""
	20	}
	21	},
	22	{
	23	"name": "Linux",
	24	"includePath": [
	25	"/usr/include/c++/4.8.4",
	26	"/usr/include/x86_64-linux-gnu/c++/4.8",
	27	"/usr/local/include",
	28	"/usr/include",
	29	"/usr/include/x86_64-linux-gnu",
	30	"${workspaceRoot}"
	31	],
	32	"defines": [],
	33	"intelliSenseMode": "clang-x64",
	34	"browse": {
	35	"path": [
	36	"/usr/include/c++/4.8.4",
	37	"/usr/include/x86_64-linux-gnu/c++/4.8",
	38	"/usr/local/include",
	39	"/usr/include",
	40	"/usr/include/x86_64-linux-gnu",
	41	"${workspaceRoot}"
	42	],
	43	"limitSymbolsToIncludedHeaders": true,
	44	"databaseFilename": ""
	45	}
	46	},
	47	{
	48	"name": "Win32",
	49	"includePath": [
	50	"C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/include",
	51	"${workspaceRoot}"
	52	],
	53	"defines": [
	54	"_DEBUG",
	55	"UNICODE"
	56	],
	57	"intelliSenseMode": "msvc-x64",
	58	"browse": {
	59	"path": [
	60	"C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/include/*",
	61	"${workspaceRoot}"
	62	],
	63	"limitSymbolsToIncludedHeaders": true,
	64	"databaseFilename": ""
	65	}
	66	}
	67	],
	68	"version": 2
	69	} No newline at end of file

.vscode/settings.json created 644 +7 0

@@ -0,0 +1,7
	1	{
	2	"files.associations": {
	3	"arrayobject.h": "c",
	4	"ndarrayobject.h": "c",
	5	"complex": "c"
	6	}
	7	} No newline at end of file

.gitignore +101 -6

@@ -1,13 +1,108
1	.project	1	# Byte-compiled / optimized / DLL files
2	.pydevproject	2	__pycache__/
3	build/	3	*.py[cod]
4	*.pyc	4	*$py.class
		5
		6	# C extensions
5	*.so	7	*.so
		8
		9	# Distribution / packaging
		10	.Python
		11	env/
		12	build/
		13	develop-eggs/
		14	dist/
		15	downloads/
		16	eggs/
		17	.eggs/
		18	lib/
		19	lib64/
		20	parts/
		21	sdist/
		22	var/
		23	wheels/
6	*.egg-info/	24	*.egg-info/
		25	.installed.cfg
		26	*.egg
		27
		28	# PyInstaller
		29	# Usually these files are written by a python script from a template
		30	# before PyInstaller builds the exe, so as to inject date/other infos into it.
		31	*.manifest
		32	*.spec
		33
		34	# Installer logs
		35	pip-log.txt
		36	pip-delete-this-directory.txt
		37
		38	# Unit test / coverage reports
		39	htmlcov/
		40	.tox/
		41	.coverage
		42	.coverage.*
		43	.cache
		44	nosetests.xml
		45	coverage.xml
		46	*,cover
		47	.hypothesis/
		48
		49	# Translations
		50	*.mo
		51	*.pot
		52
		53	# Django stuff:
		54	*.log
		55	local_settings.py
		56
		57	# Flask stuff:
		58	instance/
		59	.webassets-cache
		60
		61	# Scrapy stuff:
		62	.scrapy
		63
		64	# Sphinx documentation
		65	docs/_build/
		66
		67	# PyBuilder
		68	target/
		69
		70	# Jupyter Notebook
		71	.ipynb_checkpoints
		72
		73	# pyenv
		74	.python-version
		75
		76	# celery beat schedule file
		77	celerybeat-schedule
		78
		79	# SageMath parsed files
		80	*.sage.py
		81
		82	# dotenv
		83	.env
		84
		85	# virtualenv
		86	.venv
		87	venv/
		88	ENV/
		89
		90	# Spyder project settings
		91	.spyderproject
		92	.spyproject
		93
		94	# Rope project settings
		95	.ropeproject
		96
		97	# mkdocs documentation
		98	/site
7		99
8	# eclipse	100	# eclipse
9	.project	101	.project
10	.pydevproject	102	.pydevproject
		103	.svn/
		104	*.png
		105	*.pyc
		106	schainpy/scripts
11		107
12	# vscode	108	schaingui/node_modules No newline at end of file
13	.vscode No newline at end of file

schainpy/controller.py +31 -31

             '''
             Created on September , 2012
             @author:
             '''
             import sys
             import ast
             import datetime
             import traceback
             import schainpy
             import schainpy.admin
             from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
             from xml.dom import minidom
             from schainpy.model import *
             from time import sleep
             def prettify(elem):
                 """Return a pretty-printed XML string for the Element.
                 """
                 rough_string = tostring(elem, 'utf-8')
                 reparsed = minidom.parseString(rough_string)
                 return reparsed.toprettyxml(indent="  ")
             class ParameterConf():
                 id = None
                 name = None
                 value = None
                 format = None
                 __formated_value = None
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.format = 'str'
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getValue(self):
                     value = self.value
                     format = self.format
                     if self.__formated_value != None:
                         return self.__formated_value
                     if format == 'str':
                         self.__formated_value = str(value)
                         return self.__formated_value
                     if value == '':
                         raise ValueError, "%s: This parameter value is empty" %self.name
                     if format == 'list':
                         strList = value.split(',')
                         self.__formated_value = strList
                         return self.__formated_value
                     if format == 'intlist':
                         """
                         Example:
                             value = (0,1,2)
                         """
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             new_value = [int(new_value)]
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'floatlist':
                         """
                         Example:
                             value = (0.5, 1.4, 2.7)
                         """
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             new_value = [float(new_value)]
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'date':
                         strList = value.split('/')
                         intList = [int(x) for x in strList]
                         date = datetime.date(intList[0], intList[1], intList[2])
                         self.__formated_value = date
                         return self.__formated_value
                     if format == 'time':
                         strList = value.split(':')
                         intList = [int(x) for x in strList]
                         time = datetime.time(intList[0], intList[1], intList[2])
                         self.__formated_value = time
                         return self.__formated_value
                     if format == 'pairslist':
                         """
                         Example:
                             value = (0,1),(1,2)
                         """
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             raise ValueError, "%s has to be a tuple or list of pairs" %value
                         if type(new_value[0]) not in (tuple, list):
                             if len(new_value) != 2:
                                 raise ValueError, "%s has to be a tuple or list of pairs" %value
                             new_value = [new_value]
                         for thisPair in new_value:
                             if len(thisPair) != 2:
                                 raise ValueError, "%s has to be a tuple or list of pairs" %value
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'multilist':
                         """
                         Example:
                             value = (0,1,2),(3,4,5)
                         """
                         multiList = ast.literal_eval(value)
                         if type(multiList[0]) == int:
                             multiList = ast.literal_eval("(" + value + ")")
                         self.__formated_value = multiList
                         return self.__formated_value
                     if format == 'bool':
                         value = int(value)
                     if format == 'int':
                         value = float(value)
                     format_func = eval(format)
                     self.__formated_value = format_func(value)
                     return self.__formated_value
                 def updateId(self, new_id):
                     self.id = str(new_id)
                 def setup(self, id, name, value, format='str'):
                     self.id = str(id)
                     self.name = name
                     self.value = str(value)
                     self.format = str.lower(format)
                     self.getValue()
                     return 1
                 def update(self, name, value, format='str'):
                     self.name = name
                     self.value = str(value)
                     self.format = format
                 def makeXml(self, opElement):
                     parmElement = SubElement(opElement, self.ELEMENTNAME)
                     parmElement.set('id', str(self.id))
                     parmElement.set('name', self.name)
                     parmElement.set('value', self.value)
                     parmElement.set('format', self.format)
                 def readXml(self, parmElement):
                     self.id = parmElement.get('id')
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                     self.format = str.lower(parmElement.get('format'))
                     #Compatible with old signal chain version
                     if self.format == 'int' and self.name == 'idfigure':
                         self.name = 'id'
                 def printattr(self):
                     print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
             class OperationConf():
                 id = None
                 name = None
                 priority = None
                 type = None
                 parmConfObjList = []
                 ELEMENTNAME = 'Operation'
                 def __init__(self):
                     self.id = '0'
                     self.name = None
                     self.priority = None
                     self.type = 'self'
                 def __getNewId(self):
                     return int(self.id)*10 + len(self.parmConfObjList) + 1
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     n = 1
                     for parmObj in self.parmConfObjList:
                         idParm = str(int(new_id)*10 + n)
                         parmObj.updateId(idParm)
                         n += 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getParameterObjList(self):
                     return self.parmConfObjList
                 def getParameterObj(self, parameterName):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.name != parameterName:
                             continue
                         return parmConfObj
                     return None
                 def getParameterObjfromValue(self, parameterValue):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.getValue() != parameterValue:
                             continue
                         return parmConfObj.getValue()
                     return None
                 def getParameterValue(self, parameterName):
                     parameterObj = self.getParameterObj(parameterName)
-            #         if not parameterObj:
+                #         if not parameterObj:
-            #             return None
+                #             return None
                     value = parameterObj.getValue()
                     return value
                 def setup(self, id, name, priority, type):
                     self.id = str(id)
                     self.name = name
                     self.type = type
                     self.priority = priority
                     self.parmConfObjList = []
                 def removeParameters(self):
                     for obj in self.parmConfObjList:
                         del obj
                     self.parmConfObjList = []
                 def addParameter(self, name, value, format='str'):
                     id = self.__getNewId()
                     parmConfObj = ParameterConf()
                     if not parmConfObj.setup(id, name, value, format):
                         return None
                     self.parmConfObjList.append(parmConfObj)
                     return parmConfObj
                 def changeParameter(self, name, value, format='str'):
                     parmConfObj = self.getParameterObj(name)
                     parmConfObj.update(name, value, format)
                     return parmConfObj
                 def makeXml(self, procUnitElement):
                     opElement = SubElement(procUnitElement, self.ELEMENTNAME)
                     opElement.set('id', str(self.id))
                     opElement.set('name', self.name)
                     opElement.set('type', self.type)
                     opElement.set('priority', str(self.priority))
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.makeXml(opElement)
                 def readXml(self, opElement):
                     self.id = opElement.get('id')
                     self.name = opElement.get('name')
                     self.type = opElement.get('type')
                     self.priority = opElement.get('priority')
                     #Compatible with old signal chain version
                     #Use of 'run' method instead 'init'
                     if self.type == 'self' and self.name == 'init':
                         self.name = 'run'
                     self.parmConfObjList = []
                     parmElementList = opElement.iter(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         #Compatible with old signal chain version
                         #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
                         if self.type != 'self' and self.name == 'Plot':
                             if parmConfObj.format == 'str' and parmConfObj.name == 'type':
                                 self.name = parmConfObj.value
                                 continue
                         self.parmConfObjList.append(parmConfObj)
                 def printattr(self):
                     print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
                                                                           self.id,
                                                                           self.name,
                                                                           self.type,
                                                                           self.priority)
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.printattr()
                 def createObject(self, plotter_queue=None):
                     if self.type == 'self':
                         raise ValueError, "This operation type cannot be created"
                     if self.type == 'plotter':
                         #Plotter(plotter_name)
                         if not plotter_queue:
                             raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
                         opObj = Plotter(self.name, plotter_queue)
                     if self.type == 'external' or self.type == 'other':
                         className = eval(self.name)
                         opObj = className()
                     return opObj
             class ProcUnitConf():
                 id = None
                 name = None
                 datatype = None
                 inputId = None
                 parentId = None
                 opConfObjList = []
                 procUnitObj = None
                 opObjList = []
                 ELEMENTNAME = 'ProcUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.opConfObjList = []
                     self.procUnitObj = None
                     self.opObjDict = {}
                 def __getPriority(self):
                     return len(self.opConfObjList)+1
                 def __getNewId(self):
                     return int(self.id)*10 + len(self.opConfObjList) + 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def updateId(self, new_id, parentId=parentId):
                     new_id = int(parentId)*10 + (int(self.id) % 10)
                     new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
                     #If this proc unit has not inputs
                     if self.inputId == '0':
                         new_inputId = 0
                     n = 1
                     for opConfObj in self.opConfObjList:
                         idOp = str(int(new_id)*10 + n)
                         opConfObj.updateId(idOp)
                         n += 1
                     self.parentId = str(parentId)
                     self.id = str(new_id)
                     self.inputId = str(new_inputId)
                 def getInputId(self):
                     return self.inputId
                 def getOperationObjList(self):
                     return self.opConfObjList
                 def getOperationObj(self, name=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.name != name:
                             continue
                         return opConfObj
                     return None
                 def getOpObjfromParamValue(self, value=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
                             continue
                         return opConfObj
                     return None
                 def getProcUnitObj(self):
                     return self.procUnitObj
                 def setup(self, id, name, datatype, inputId, parentId=None):
                     #Compatible with old signal chain version
                     if datatype==None and name==None:
                         raise ValueError, "datatype or name should be defined"
                     if name==None:
                         if 'Proc' in datatype:
                             name = datatype
                         else:
                             name = '%sProc' %(datatype)
                     if datatype==None:
                         datatype = name.replace('Proc','')
                     self.id = str(id)
                     self.name = name
                     self.datatype = datatype
                     self.inputId = inputId
                     self.parentId = parentId
                     self.opConfObjList = []
                     self.addOperation(name='run', optype='self')
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                     self.addOperation(name='run')
                 def addParameter(self, **kwargs):
                     '''
                     Add parameters to "run" operation
                     '''
                     opObj = self.opConfObjList[0]
                     opObj.addParameter(**kwargs)
                     return opObj
                 def addOperation(self, name, optype='self'):
                     id = self.__getNewId()
                     priority = self.__getPriority()
                     opConfObj = OperationConf()
                     opConfObj.setup(id, name=name, priority=priority, type=optype)
                     self.opConfObjList.append(opConfObj)
                     return opConfObj
                 def makeXml(self, projectElement):
                     procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
                     procUnitElement.set('id', str(self.id))
                     procUnitElement.set('name', self.name)
                     procUnitElement.set('datatype', self.datatype)
                     procUnitElement.set('inputId', str(self.inputId))
                     for opConfObj in self.opConfObjList:
                         opConfObj.makeXml(procUnitElement)
                 def readXml(self, upElement):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.inputId = upElement.get('inputId')
                     if self.ELEMENTNAME == "ReadUnit":
                         self.datatype = self.datatype.replace("Reader", "")
                     if self.ELEMENTNAME == "ProcUnit":
                         self.datatype = self.datatype.replace("Proc", "")
                     if self.inputId == 'None':
                         self.inputId = '0'
                     self.opConfObjList = []
                     opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement)
                         self.opConfObjList.append(opConfObj)
                 def printattr(self):
                     print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
-                                                                         self.id,
+                                                                            self.id,
-                                                                         self.name,
+                                                                            self.name,
-                                                                         self.datatype,
+                                                                            self.datatype,
-                                                                         self.inputId)
+                                                                            self.inputId)
                     for opConfObj in self.opConfObjList:
                         opConfObj.printattr()
                 def createObjects(self, plotter_queue=None):
                     className = eval(self.name)
                     procUnitObj = className()
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self':
                             continue
                         opObj = opConfObj.createObject(plotter_queue)
                         self.opObjDict[opConfObj.id] = opObj
                         procUnitObj.addOperation(opObj, opConfObj.id)
                     self.procUnitObj = procUnitObj
                     return procUnitObj
                 def run(self):
                     is_ok = False
                     for opConfObj in self.opConfObjList:
                         kwargs = {}
                         for parmConfObj in opConfObj.getParameterObjList():
                             if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
                                 continue
                             kwargs[parmConfObj.name] = parmConfObj.getValue()
                         #ini = time.time()
                         #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
                         sts = self.procUnitObj.call(opType = opConfObj.type,
                                                     opName = opConfObj.name,
                                                     opId = opConfObj.id,
-                                                     **kwargs)
+                                                    **kwargs)
-            #             total_time = time.time() - ini
-            #             if total_time > 0.002:
-            #                 print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
+                        #             total_time = time.time() - ini
+                        #
+                        #             if total_time > 0.002:
+                        #                 print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
                         is_ok = is_ok or sts
                     return is_ok
                 def close(self):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self':
                             continue
                         opObj = self.procUnitObj.getOperationObj(opConfObj.id)
                         opObj.close()
                     self.procUnitObj.close()
                     return
             class ReadUnitConf(ProcUnitConf):
                 path = None
                 startDate = None
                 endDate = None
                 startTime = None
                 endTime = None
                 ELEMENTNAME = 'ReadUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.parentId = None
                     self.opConfObjList = []
                     self.opObjList = []
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, **kwargs):
                     #Compatible with old signal chain version
                     if datatype==None and name==None:
                         raise ValueError, "datatype or name should be defined"
                     if name==None:
                         if 'Reader' in datatype:
                             name = datatype
                         else:
                             name = '%sReader' %(datatype)
                     if datatype==None:
                         datatype = name.replace('Reader','')
                     self.id = id
                     self.name = name
                     self.datatype = datatype
                     self.path = os.path.abspath(path)
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.inputId = '0'
                     self.parentId = parentId
                     self.addRunOperation(**kwargs)
                 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
                     #Compatible with old signal chain version
                     if datatype==None and name==None:
                         raise ValueError, "datatype or name should be defined"
                     if name==None:
                         if 'Reader' in datatype:
                             name = datatype
                         else:
                             name = '%sReader' %(datatype)
                     if datatype==None:
                         datatype = name.replace('Reader','')
                     self.datatype = datatype
                     self.name = name
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.inputId = '0'
                     self.parentId = parentId
                     self.updateRunOperation(**kwargs)
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                 def addRunOperation(self, **kwargs):
                     opObj = self.addOperation(name = 'run', optype = 'self')
                     opObj.addParameter(name='datatype' , value=self.datatype, format='str')
                     opObj.addParameter(name='path'     , value=self.path, format='str')
                     opObj.addParameter(name='startDate' , value=self.startDate, format='date')
                     opObj.addParameter(name='endDate'   , value=self.endDate, format='date')
                     opObj.addParameter(name='startTime' , value=self.startTime, format='time')
                     opObj.addParameter(name='endTime'   , value=self.endTime, format='time')
                     for key, value in kwargs.items():
                         opObj.addParameter(name=key, value=value, format=type(value).__name__)
                     return opObj
                 def updateRunOperation(self, **kwargs):
                     opObj = self.getOperationObj(name = 'run')
                     opObj.removeParameters()
                     opObj.addParameter(name='datatype' , value=self.datatype, format='str')
                     opObj.addParameter(name='path'     , value=self.path, format='str')
                     opObj.addParameter(name='startDate' , value=self.startDate, format='date')
                     opObj.addParameter(name='endDate'   , value=self.endDate, format='date')
                     opObj.addParameter(name='startTime' , value=self.startTime, format='time')
                     opObj.addParameter(name='endTime'   , value=self.endTime, format='time')
                     for key, value in kwargs.items():
                         opObj.addParameter(name=key, value=value, format=type(value).__name__)
                     return opObj
-            #     def makeXml(self, projectElement):
+                    #     def makeXml(self, projectElement):
+                    #
-            #         procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
+                    #         procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
-            #         procUnitElement.set('id', str(self.id))
+                    #         procUnitElement.set('id', str(self.id))
-            #         procUnitElement.set('name', self.name)
+                    #         procUnitElement.set('name', self.name)
-            #         procUnitElement.set('datatype', self.datatype)
+                    #         procUnitElement.set('datatype', self.datatype)
-            #         procUnitElement.set('inputId', str(self.inputId))
+                    #         procUnitElement.set('inputId', str(self.inputId))
+                    #
-            #         for opConfObj in self.opConfObjList:
+                    #         for opConfObj in self.opConfObjList:
-            #             opConfObj.makeXml(procUnitElement)
+                    #             opConfObj.makeXml(procUnitElement)
                 def readXml(self, upElement):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.inputId = upElement.get('inputId')
                     if self.ELEMENTNAME == "ReadUnit":
                         self.datatype = self.datatype.replace("Reader", "")
                     if self.inputId == 'None':
                         self.inputId = '0'
                     self.opConfObjList = []
                     opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement)
                         self.opConfObjList.append(opConfObj)
                         if opConfObj.name == 'run':
                             self.path = opConfObj.getParameterValue('path')
                             self.startDate = opConfObj.getParameterValue('startDate')
                             self.endDate = opConfObj.getParameterValue('endDate')
                             self.startTime = opConfObj.getParameterValue('startTime')
                             self.endTime = opConfObj.getParameterValue('endTime')
             class Project():
                 id = None
                 name = None
                 description = None
                 filename = None
                 procUnitConfObjDict = None
                 ELEMENTNAME = 'Project'
                 plotterQueue = None
                 def __init__(self, plotter_queue=None):
                     self.id = None
                     self.name = None
                     self.description = None
                     self.plotterQueue = plotter_queue
                     self.procUnitConfObjDict = {}
                 def __getNewId(self):
                     idList = self.procUnitConfObjDict.keys()
                     id = int(self.id)*10
                     while True:
                         id += 1
                         if str(id) in idList:
                             continue
                         break
                     return str(id)
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     keyList = self.procUnitConfObjDict.keys()
                     keyList.sort()
                     n = 1
                     newProcUnitConfObjDict = {}
                     for procKey in keyList:
                         procUnitConfObj = self.procUnitConfObjDict[procKey]
                         idProcUnit = str(int(self.id)*10 + n)
                         procUnitConfObj.updateId(idProcUnit, parentId = self.id)
                         newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
                         n += 1
                     self.procUnitConfObjDict = newProcUnitConfObjDict
                 def setup(self, id, name, description):
                     self.id = str(id)
                     self.name = name
                     self.description = description
                 def update(self, name, description):
                     self.name = name
                     self.description = description
                 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
                     if id is None:
                         idReadUnit = self.__getNewId()
                     else:
                         idReadUnit = str(id)
                     readUnitConfObj = ReadUnitConf()
                     readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
                     self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     return readUnitConfObj
                 def addProcUnit(self, inputId='0', datatype=None, name=None):
                     idProcUnit = self.__getNewId()
                     procUnitConfObj = ProcUnitConf()
                     procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
                     self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     return procUnitConfObj
                 def removeProcUnit(self, id):
                     if id in self.procUnitConfObjDict.keys():
                         self.procUnitConfObjDict.pop(id)
                 def getReadUnitId(self):
                     readUnitConfObj = self.getReadUnitObj()
                     return readUnitConfObj.id
                 def getReadUnitObj(self):
                     for obj in self.procUnitConfObjDict.values():
                         if obj.getElementName() == "ReadUnit":
                             return obj
                     return None
                 def getProcUnitObj(self, id=None, name=None):
                     if id != None:
                         return self.procUnitConfObjDict[id]
                     if name != None:
                         return self.getProcUnitObjByName(name)
                     return None
                 def getProcUnitObjByName(self, name):
                     for obj in self.procUnitConfObjDict.values():
                         if obj.name == name:
                             return obj
                     return None
                 def procUnitItems(self):
                     return self.procUnitConfObjDict.items()
                 def makeXml(self):
                     projectElement = Element('Project')
                     projectElement.set('id', str(self.id))
                     projectElement.set('name', self.name)
                     projectElement.set('description', self.description)
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.makeXml(projectElement)
                     self.projectElement = projectElement
                 def writeXml(self, filename=None):
                     if filename == None:
                         if self.filename:
                             filename = self.filename
                         else:
                             filename = "schain.xml"
                     if not filename:
                         print "filename has not been defined. Use setFilename(filename) for do it."
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.access(os.path.dirname(abs_file), os.W_OK):
                         print "No write permission on %s" %os.path.dirname(abs_file)
                         return 0
                     if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
                         print "File %s already exists and it could not be overwriten" %abs_file
                         return 0
                     self.makeXml()
                     ElementTree(self.projectElement).write(abs_file, method='xml')
                     self.filename = abs_file
                     return 1
                 def readXml(self, filename = None):
                     if not filename:
                         print "filename is not defined"
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.path.isfile(abs_file):
                         print "%s file does not exist" %abs_file
                         return 0
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     try:
                         self.projectElement = ElementTree().parse(abs_file)
                     except:
                         print "Error reading %s, verify file format" %filename
                         return 0
                     self.project = self.projectElement.tag
                     self.id = self.projectElement.get('id')
                     self.name = self.projectElement.get('name')
                     self.description = self.projectElement.get('description')
                     readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
                     for readUnitElement in readUnitElementList:
                         readUnitConfObj = ReadUnitConf()
                         readUnitConfObj.readXml(readUnitElement)
                         if readUnitConfObj.parentId == None:
                             readUnitConfObj.parentId = self.id
                         self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
                     for procUnitElement in procUnitElementList:
                         procUnitConfObj = ProcUnitConf()
                         procUnitConfObj.readXml(procUnitElement)
                         if procUnitConfObj.parentId == None:
                             procUnitConfObj.parentId = self.id
                         self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     self.filename = abs_file
                     return 1
                 def printattr(self):
                     print "Project[%s]: name = %s, description = %s" %(self.id,
-                                                                          self.name,
+                                                                    self.name,
-                                                                          self.description)
+                                                                    self.description)
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.printattr()
                 def createObjects(self):
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.createObjects(self.plotterQueue)
                 def __connect(self, objIN, thisObj):
                     thisObj.setInput(objIN.getOutputObj())
                 def connectObjects(self):
                     for thisPUConfObj in self.procUnitConfObjDict.values():
                         inputId = thisPUConfObj.getInputId()
                         if int(inputId) == 0:
                             continue
                         #Get input object
                         puConfINObj = self.procUnitConfObjDict[inputId]
                         puObjIN = puConfINObj.getProcUnitObj()
                         #Get current object
                         thisPUObj = thisPUConfObj.getProcUnitObj()
                         self.__connect(puObjIN, thisPUObj)
                 def __handleError(self, procUnitConfObj, send_email=True):
                     import socket
                     err = traceback.format_exception(sys.exc_info()[0],
-                                                     sys.exc_info()[1],
+                                                    sys.exc_info()[1],
-                                                     sys.exc_info()[2])
+                                                    sys.exc_info()[2])
                     print "***** Error occurred in %s *****" %(procUnitConfObj.name)
                     print "***** %s" %err[-1]
                     message = "".join(err)
                     sys.stderr.write(message)
                     if not send_email:
                         return
                     subject =  "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
                     subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
                     subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
                     subtitle += "Working directory: %s\n" %os.path.abspath("./")
                     subtitle += "Configuration file: %s\n" %self.filename
                     subtitle += "Time: %s\n" %str(datetime.datetime.now())
                     readUnitConfObj = self.getReadUnitObj()
                     if readUnitConfObj:
                         subtitle += "\nInput parameters:\n"
                         subtitle += "[Data path = %s]\n" %readUnitConfObj.path
                         subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
                         subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
                         subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
                         subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
                         subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
                     adminObj = schainpy.admin.SchainNotify()
                     adminObj.sendAlert(message=message,
-                                       subject=subject,
+                                    subject=subject,
-                                       subtitle=subtitle,
+                                    subtitle=subtitle,
-                                       filename=self.filename)
+                                    filename=self.filename)
                 def isPaused(self):
                     return 0
                 def isStopped(self):
                     return 0
                 def runController(self):
                     """
                     returns 0 when this process has been stopped, 1 otherwise
                     """
                     if self.isPaused():
                         print "Process suspended"
                         while True:
                             sleep(0.1)
                             if not self.isPaused():
                                 break
                             if self.isStopped():
                                 break
                         print "Process reinitialized"
                     if self.isStopped():
                         print "Process stopped"
                         return 0
                     return 1
                 def setFilename(self, filename):
                     self.filename = filename
                 def setPlotterQueue(self, plotter_queue):
                     raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
                 def getPlotterQueue(self):
                     raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
                 def useExternalPlotter(self):
                     raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
                 def run(self):
                     print
                     print "*"*60
                     print "   Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
                     print "*"*60
                     print
                     keyList = self.procUnitConfObjDict.keys()
                     keyList.sort()
                     while(True):
                         is_ok = False
                         for procKey in keyList:
             #                 print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
                             procUnitConfObj = self.procUnitConfObjDict[procKey]
                             try:
                                 sts = procUnitConfObj.run()
                                 is_ok = is_ok or sts
                             except KeyboardInterrupt:
                                 is_ok = False
                                 break
                             except ValueError, e:
                                 sleep(0.5)
                                 self.__handleError(procUnitConfObj, send_email=True)
                                 is_ok = False
                                 break
                             except:
                                 sleep(0.5)
                                 self.__handleError(procUnitConfObj)
                                 is_ok = False
                                 break
                         #If every process unit finished so end process
                         if not(is_ok):
             #                 print "Every process unit have finished"
                             break
                         if not self.runController():
                             break
                     #Closing every process
                     for procKey in keyList:
                         procUnitConfObj = self.procUnitConfObjDict[procKey]
                         procUnitConfObj.close()
                     print "Process finished"
                 def start(self):
                     self.writeXml()
                     self.createObjects()
                     self.connectObjects()
                     self.run()
             if __name__ == '__main__':
                 desc = "Segundo Test"
                 filename = "schain.xml"
                 controllerObj = Project()
                 controllerObj.setup(id = '191', name='test01', description=desc)
                 readUnitConfObj = controllerObj.addReadUnit(datatype='Voltage',
                                                             path='data/rawdata/',
                                                             startDate='2011/01/01',
                                                             endDate='2012/12/31',
                                                             startTime='00:00:00',
                                                             endTime='23:59:59',
                                                             online=1,
                                                             walk=1)
                 procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage', inputId=readUnitConfObj.getId())
                 opObj10 = procUnitConfObj0.addOperation(name='selectChannels')
                 opObj10.addParameter(name='channelList', value='3,4,5', format='intlist')
                 opObj10 = procUnitConfObj0.addOperation(name='selectHeights')
                 opObj10.addParameter(name='minHei', value='90', format='float')
                 opObj10.addParameter(name='maxHei', value='180', format='float')
                 opObj12 = procUnitConfObj0.addOperation(name='CohInt', optype='external')
                 opObj12.addParameter(name='n', value='10', format='int')
                 procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj0.getId())
                 procUnitConfObj1.addParameter(name='nFFTPoints', value='32', format='int')
             #    procUnitConfObj1.addParameter(name='pairList', value='(0,1),(0,2),(1,2)', format='')
                 opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
                 opObj11.addParameter(name='idfigure', value='1', format='int')
                 opObj11.addParameter(name='wintitle', value='SpectraPlot0', format='str')
                 opObj11.addParameter(name='zmin', value='40', format='int')
                 opObj11.addParameter(name='zmax', value='90', format='int')
                 opObj11.addParameter(name='showprofile', value='1', format='int')
                 print "Escribiendo el archivo XML"
                 controllerObj.writeXml(filename)
                 print "Leyendo el archivo XML"
                 controllerObj.readXml(filename)
                 #controllerObj.printattr()
                 controllerObj.createObjects()
                 controllerObj.connectObjects()
                 controllerObj.run()
   No newline at end of file

schainpy/controller_api.py +2 -2

             import threading
             from Queue import Queue
             from schainpy.controller import Project
             from schainpy.model.graphics.jroplotter import PlotManager
             class ControllerThread(threading.Thread, Project):
                 def __init__(self, plotter_queue=None):
                     threading.Thread.__init__(self)
                     Project.__init__(self, plotter_queue)
                     self.setDaemon(True)
                     self.lock = threading.Lock()
-                    self.control = {'stop':False, 'pause':False}
+                    self.control = { 'stop':False, 'pause':False }
                 def __del__(self):
                     self.control['stop'] = True
                 def stop(self):
                     self.lock.acquire()
                     self.control['stop'] = True
                     self.lock.release()
                 def pause(self):
                     self.lock.acquire()
                     self.control['pause'] = not(self.control['pause'])
                     paused = self.control['pause']
                     self.lock.release()
                     return paused
                 def isPaused(self):
                     self.lock.acquire()
                     paused = self.control['pause']
                     self.lock.release()
                     return paused
                 def isStopped(self):
                     self.lock.acquire()
                     stopped = self.control['stop']
                     self.lock.release()
                     return stopped
                 def run(self):
                     self.control['stop'] = False
                     self.control['pause'] = False
                     self.writeXml()
                     self.createObjects()
                     self.connectObjects()
                     Project.run(self)
                 def isRunning(self):
                     return self.is_alive()
                 def isFinished(self):
                     return not self.is_alive()
                 def setPlotters(self):
                     plotterList = PlotManager.plotterList
                     for thisPUConfObj in self.procUnitConfObjDict.values():
                         inputId = thisPUConfObj.getInputId()
                         if int(inputId) == 0:
                             continue
                         for thisOpObj in thisPUConfObj.getOperationObjList():
                             if thisOpObj.type == "self":
                                 continue
                             if thisOpObj.name in plotterList:
-                                thisOpObj.type = "plotter"
+                                thisOpObj.type = "other"
                 def setPlotterQueue(self, plotter_queue):
                     self.plotterQueue = plotter_queue
                 def getPlotterQueue(self):
                     return self.plotterQueue
                 def useExternalPlotter(self):
                     self.plotterQueue = Queue(10)
                     self.setPlotters()
                     plotManagerObj = PlotManager(self.plotterQueue)
                     plotManagerObj.setController(self)
                     return plotManagerObj
             # from PyQt4 import QtCore
             # from PyQt4.QtCore import SIGNAL
             #
             # class ControllerQThread(QtCore.QThread, Project):
             #
             #     def __init__(self, filename):
             #
             #         QtCore.QThread.__init__(self)
             #         Project.__init__(self)
             #
             #         self.filename = filename
             #
             #         self.lock = threading.Lock()
             #         self.control = {'stop':False, 'pause':False}
             #
             #     def __del__(self):
             #
             #         self.control['stop'] = True
             #         self.wait()
             #
             #     def stop(self):
             #
             #         self.lock.acquire()
             #
             #         self.control['stop'] = True
             #
             #         self.lock.release()
             #
             #     def pause(self):
             #
             #         self.lock.acquire()
             #
             #         self.control['pause'] = not(self.control['pause'])
             #         paused = self.control['pause']
             #
             #         self.lock.release()
             #
             #         return paused
             #
             #     def isPaused(self):
             #
             #         self.lock.acquire()
             #         paused = self.control['pause']
             #         self.lock.release()
             #
             #         return paused
             #
             #     def isStopped(self):
             #
             #         self.lock.acquire()
             #         stopped = self.control['stop']
             #         self.lock.release()
             #
             #         return stopped
             #
             #     def run(self):
             #
             #         self.control['stop'] = False
             #         self.control['pause'] = False
             #
             #         self.readXml(self.filename)
             #         self.createObjects()
             #         self.connectObjects()
             #         self.emit( SIGNAL( "jobStarted( PyQt_PyObject )" ), 1)
             #         Project.run(self)
             #         self.emit( SIGNAL( "jobFinished( PyQt_PyObject )" ), 1)
             #

schainpy/model/utils/jroutils_ftp.py +6 -2

             '''
             @author: Daniel Suarez
             '''
             import os
             import glob
             import ftplib
             try:
                 import paramiko
                 import scp
             except:
                 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
             import time
             import threading
             Thread = threading.Thread
             # try:
             #     from gevent import sleep
             # except:
             from time import sleep
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             class Remote(Thread):
                 """
                 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
                 used to upload or download files remotely.
                 Non-standard Python modules used:
                     None
                 Written by:
                     "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe  Jun. 03, 2015
                 """
                 server = None
                 username = None
                 password = None
                 remotefolder = None
                 period = 60
                 fileList = []
                 bussy = False
                 def __init__(self, server, username, password, remotefolder, period=60):
                     Thread.__init__(self)
                     self.setDaemon(True)
                     self.status = 0
                     self.__server = server
                     self.__username = username
                     self.__password = password
                     self.__remotefolder = remotefolder
                     self.period = period
                     self.fileList = []
                     self.bussy = False
                     self.stopFlag = False
                     print "[Remote Server] Opening server: %s" %self.__server
                     if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
                         print "[Remote Server] %s server was opened successfully" %self.__server
                     self.close()
                     self.mutex = threading.Lock()
                 def stop(self):
                     self.stopFlag = True
                     self.join(10)
                 def open(self):
                     """
                     Connect to server and create a connection class (FTP or SSH) to remote server.
                     """
                     raise NotImplementedError, "Implement this method in child class"
                 def close(self):
                     """
                     Close connection to server
                     """
                     raise NotImplementedError, "Implement this method in child class"
                 def mkdir(self, remotefolder):
                     """
                     Create a folder remotely
                     """
                     raise NotImplementedError, "Implement this method in child class"
                 def cd(self, remotefolder):
                     """
                     Change working directory in remote server
                     """
                     raise NotImplementedError, "Implement this method in child class"
                 def download(self, filename, localfolder=None):
                     """
                     Download a file from server to local host
                     """
                     raise NotImplementedError, "Implement this method in child class"
                 def sendFile(self, fullfilename):
                     """
                     sendFile method is used to upload a local file to the current directory in remote server
                     Inputs:
                         fullfilename        - full path name of local file to store in remote directory
                     Returns:
 in error case else 1
                     """
                     raise NotImplementedError, "Implement this method in child class"
                 def upload(self, fullfilename, remotefolder=None):
                     """
                     upload method is used to upload a local file to remote directory. This method changes
                     working directory before sending a file.
                     Inputs:
                         fullfilename        - full path name of local file to store in remote directory
                         remotefolder    - remote directory
                     Returns:
 in error case else 1
                     """
                     print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
                     if not self.status:
                         return 0
                     if remotefolder == None:
                         remotefolder = self.remotefolder
                     if not self.cd(remotefolder):
                         return 0
                     if not self.sendFile(fullfilename):
                         print "[Remote Server] Error uploading file %s" %fullfilename
                         return 0
                     print "[Remote Server] upload finished successfully"
                     return 1
                 def delete(self, filename):
                     """
                     Remove a file from remote server
                     """
                     pass
                 def updateFileList(self, fileList):
                     """
                     Remove a file from remote server
                     """
                     if fileList == self.fileList:
                         return 0
                     self.mutex.acquire()
             #         init = time.time()
             #
             #         while(self.bussy):
             #             sleep(0.1)
             #             if time.time() - init > 2*self.period:
             #                 return 0
                     self.fileList = fileList
                     self.mutex.release()
                     return 1
                 def run(self):
                     if not self.status:
                         print "Finishing FTP service"
                         return
                     if not self.cd(self.remotefolder):
                         raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
                     while True:
                         for i in range(self.period):
                             if self.stopFlag:
                                 break
                             sleep(1)
                         if self.stopFlag:
                             break
             #             self.bussy = True
                         self.mutex.acquire()
                         print "[Remote Server] Opening %s" %self.__server
                         if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
                             self.mutex.release()
                             continue
                         for thisFile in self.fileList:
                             self.upload(thisFile, self.remotefolder)
                         print "[Remote Server] Closing %s" %self.__server
                         self.close()
                         self.mutex.release()
             #             self.bussy = False
                     print "[Remote Server] Thread stopped successfully"
             class FTPClient(Remote):
                 __ftpClientObj = None
                 def __init__(self, server, username, password, remotefolder, period=60):
                     """
                     """
                     Remote.__init__(self, server, username, password, remotefolder, period)
                 def open(self, server, username, password, remotefolder):
                     """
                     This method is used to set FTP parameters and establish a connection to remote server
                     Inputs:
                         server    - remote server IP Address
                         username    - remote server Username
                         password    - remote server password
                         remotefolder    - remote server current working directory
                     Return:
                         Boolean     -     Returns 1 if a connection has been established, 0 otherwise
                     Affects:
                         self.status        - in case of error or fail connection this parameter is set to 0 else 1
                     """
                     if server == None:
                         raise ValueError, "FTP server should be defined"
                     if username == None:
                         raise ValueError, "FTP username should be defined"
                     if password == None:
                         raise ValueError, "FTP password should be defined"
                     if remotefolder == None:
                         raise ValueError, "FTP remote folder should be defined"
                     try:
                         ftpClientObj = ftplib.FTP(server)
                     except ftplib.all_errors, e:
                         print "[FTP Server]: FTP server connection fail: %s" %server
                         print "[FTP Server]:", e
                         self.status = 0
                         return 0
                     try:
                         ftpClientObj.login(username, password)
                     except ftplib.all_errors:
                         print "[FTP Server]: FTP username or password are incorrect"
                         self.status = 0
                         return 0
                     if remotefolder == None:
                         remotefolder = ftpClientObj.pwd()
                     else:
                         try:
                             ftpClientObj.cwd(remotefolder)
                         except ftplib.all_errors:
                             print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
                             remotefolder = ftpClientObj.pwd()
                     self.server = server
                     self.username = username
                     self.password = password
                     self.remotefolder = remotefolder
                     self.__ftpClientObj = ftpClientObj
                     self.status = 1
                     return 1
                 def close(self):
                     """
                     Close connection to remote server
                     """
                     if not self.status:
                         return 0
                     self.__ftpClientObj.close()
                 def mkdir(self, remotefolder):
                     """
                     mkdir is used to make a new directory in remote server
                     Input:
                         remotefolder    - directory name
                     Return:
 in error case else 1
                     """
                     if not self.status:
                         return 0
                     try:
                         self.__ftpClientObj.mkd(dirname)
                     except ftplib.all_errors:
                         print "[FTP Server]: Error creating remote folder: %s" %remotefolder
                         return 0
                     return 1
                 def cd(self, remotefolder):
                     """
                     cd is used to change remote working directory on server
                     Input:
                         remotefolder    - current working directory
                     Affects:
                         self.remotefolder
                     Return:
 in case of error else 1
                     """
                     if not self.status:
                         return 0
                     if remotefolder == self.remotefolder:
                         return 1
                     try:
                         self.__ftpClientObj.cwd(remotefolder)
                     except ftplib.all_errors:
                         print '[FTP Server]: Error changing to %s' %remotefolder
                         print '[FTP Server]: Trying to create remote folder'
                         if not self.mkdir(remotefolder):
                             print '[FTP Server]: Remote folder could not be created'
                             return 0
                         try:
                             self.__ftpClientObj.cwd(remotefolder)
                         except ftplib.all_errors:
                             return 0
                     self.remotefolder = remotefolder
                     return 1
                 def sendFile(self, fullfilename):
                     if not self.status:
                         return 0
                     fp = open(fullfilename, 'rb')
                     filename = os.path.basename(fullfilename)
                     command = "STOR %s" %filename
                     try:
                         self.__ftpClientObj.storbinary(command, fp)
                     except ftplib.all_errors, e:
                         print "[FTP Server]:", e
                         return 0
                     try:
                         self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
                     except ftplib.all_errors, e:
                         print "[FTP Server]:", e
                     fp.close()
                     return 1
             class SSHClient(Remote):
                 __sshClientObj = None
                 __scpClientObj = None
                 def __init__(self, server, username, password, remotefolder, period=60):
                     """
                     """
                     Remote.__init__(self, server, username, password, remotefolder, period)
                 def open(self, server, username, password, remotefolder, port=22):
                     """
                     This method is used to set SSH parameters and establish a connection to a remote server
                     Inputs:
                         server    - remote server IP Address
                         username    - remote server Username
                         password    - remote server password
                         remotefolder    - remote server current working directory
                     Return: void
                     Affects:
                         self.status        - in case of error or fail connection this parameter is set to 0 else 1
                     """
                     import socket
                     if server == None:
                         raise ValueError, "SSH server should be defined"
                     if username == None:
                         raise ValueError, "SSH username should be defined"
                     if password == None:
                         raise ValueError, "SSH password should be defined"
                     if remotefolder == None:
                         raise ValueError, "SSH remote folder should be defined"
                     sshClientObj = paramiko.SSHClient()
                     sshClientObj.load_system_host_keys()
                     sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
                     self.status = 0
                     try:
                         sshClientObj.connect(server, username=username, password=password, port=port)
                     except paramiko.AuthenticationException, e:
             #             print "SSH username or password are incorrect: %s"
                         print "[SSH Server]:", e
                         return 0
                     except SSHException, e:
                         print "[SSH Server]:", e
                         return 0
                     except socket.error:
                         self.status = 0
                         print "[SSH Server]:", e
                         return 0
                     self.status = 1
                     scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
                     if remotefolder == None:
                         remotefolder = self.pwd()
                     self.server = server
                     self.username = username
                     self.password = password
                     self.__sshClientObj = sshClientObj
                     self.__scpClientObj = scpClientObj
                     self.status = 1
                     if not self.cd(remotefolder):
                         raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
                         return 0
                     self.remotefolder = remotefolder
                     return 1
                 def close(self):
                     """
                     Close connection to remote server
                     """
                     if not self.status:
                         return 0
                     self.__scpClientObj.close()
                     self.__sshClientObj.close()
                 def __execute(self, command):
                     """
                     __execute a command on remote server
                     Input:
                         command    - Exmaple 'ls -l'
                     Return:
 in error case else 1
                     """
                     if not self.status:
                         return 0
                     stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
                     result = stderr.readlines()
                     if len(result) > 1:
                         return 0
                     result = stdout.readlines()
                     if len(result) > 1:
                         return result[0][:-1]
                     return 1
                 def mkdir(self, remotefolder):
                     """
                     mkdir is used to make a new directory in remote server
                     Input:
                         remotefolder    - directory name
                     Return:
 in error case else 1
                     """
                     command = 'mkdir %s' %remotefolder
                     return self.__execute(command)
                 def pwd(self):
                     command = 'pwd'
                     return self.__execute(command)
                 def cd(self, remotefolder):
                     """
                     cd is used to change remote working directory on server
                     Input:
                         remotefolder    - current working directory
                     Affects:
                         self.remotefolder
                     Return:
 in case of error else 1
                     """
                     if not self.status:
                         return 0
                     if remotefolder == self.remotefolder:
                         return 1
                     chk_command = "cd %s; pwd" %remotefolder
                     mkdir_command = "mkdir %s" %remotefolder
                     if not self.__execute(chk_command):
                         if not self.__execute(mkdir_command):
                             self.remotefolder = None
                             return 0
                     self.remotefolder = remotefolder
                     return 1
                 def sendFile(self, fullfilename):
                     if not self.status:
                         return 0
                     try:
                         self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
                     except scp.ScpError, e:
                         print "[SSH Server]", str(e)
                         return 0
                     remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
                     command = 'chmod 775 %s' %remotefile
                     return self.__execute(command)
             class SendToServer(ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.isConfig = False
                     self.clientObj = None
                 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
                     self.clientObj = None
                     self.localfolder = localfolder
                     self.ext = ext
                     self.period = period
                     if str.lower(protocol) == 'ftp':
                         self.clientObj = FTPClient(server, username, password, remotefolder, period)
                     if str.lower(protocol) == 'ssh':
                         self.clientObj = SSHClient(server, username, password, remotefolder, period)
                     if not self.clientObj:
                         raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
                     self.clientObj.start()
                 def findFiles(self):
                     if not type(self.localfolder) == list:
                         folderList = [self.localfolder]
                     else:
                         folderList = self.localfolder
                     #Remove duplicate items
                     folderList = list(set(folderList))
                     fullfilenameList = []
                     for thisFolder in folderList:
                         print "[Remote Server]: Searching files on %s" %thisFolder
                         filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
                         if len(filenameList) < 1:
                             continue
                         for thisFile in filenameList:
                             fullfilename = os.path.join(thisFolder, thisFile)
                             if fullfilename in fullfilenameList:
                                 continue
                             #Only files modified in the last 30 minutes are considered
                             if os.path.getmtime(fullfilename) < time.time() - 30*60:
                                 continue
                             fullfilenameList.append(fullfilename)
                     return fullfilenameList
                 def run(self, **kwargs):
                     if not self.isConfig:
                         self.init = time.time()
                         self.setup(**kwargs)
                         self.isConfig = True
+                    if not self.clientObj.is_alive():
+                        print "[Remote Server]: Restarting connection "
+                        self.setup(**kwargs)
                     if time.time() - self.init >= self.period:
                         fullfilenameList = self.findFiles()
                         if self.clientObj.updateFileList(fullfilenameList):
                             print "[Remote Server]: Sending the next files ", str(fullfilenameList)
                         self.init = time.time()
                 def close(self):
                     print "[Remote Server] Stopping thread"
                     self.clientObj.stop()
             class FTP(object):
                 """
                 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
                 Non-standard Python modules used: None
                 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe  Oct. 26, 2010
                 """
                 def __init__(self,server = None, username=None, password=None, remotefolder=None):
                     """
                     This method is used to setting parameters for FTP and establishing connection to remote server
                     Inputs:
                         server    - remote server IP Address
                         username    - remote server Username
                         password    - remote server password
                         remotefolder    - remote server current working directory
                     Return: void
                     Affects:
                         self.status    - in Error Case or Connection Failed this parameter is set to 1 else 0
                         self.folderList    - sub-folder list of remote folder
                         self.fileList    - file list of remote folder
                     """
                     if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
                         server, username, password, remotefolder = self.parmsByDefault()
                     self.server = server
                     self.username = username
                     self.password = password
                     self.remotefolder = remotefolder
                     self.file = None
                     self.ftp = None
                     self.status = 0
                     try:
                         self.ftp = ftplib.FTP(self.server)
                         self.ftp.login(self.username,self.password)
                         self.ftp.cwd(self.remotefolder)
             #            print 'Connect to FTP Server: Successfully'
                     except ftplib.all_errors:
                         print 'Error FTP Service'
                         self.status = 1
                         return
                     self.dirList = []
                     try:
                         self.dirList = self.ftp.nlst()
                     except ftplib.error_perm, resp:
                         if str(resp) == "550 No files found":
                                 print "no files in this directory"
                                 self.status = 1
                                 return
                     except ftplib.all_errors:
                         print 'Error Displaying Dir-Files'
                         self.status = 1
                         return
                     self.fileList = []
                     self.folderList = []
                     #only for test
                     for f in self.dirList:
                         name, ext = os.path.splitext(f)
                         if ext != '':
                             self.fileList.append(f)
             #                print 'filename: %s - size: %d'%(f,self.ftp.size(f))
                 def parmsByDefault(self):
                     server = 'jro-app.igp.gob.pe'
                     username = 'wmaster'
                     password = 'mst2010vhf'
                     remotefolder = '/home/wmaster/graficos'
                     return server, username, password, remotefolder
                 def mkd(self,dirname):
                     """
                     mkd is used to make directory in remote server
                     Input:
                         dirname    - directory name
                     Return:
 in error case else 0
                     """
                     try:
                         self.ftp.mkd(dirname)
                     except:
                         print 'Error creating remote folder:%s'%dirname
                         return 1
                     return 0
                 def delete(self,filename):
                     """
                     delete is used to delete file in current working directory of remote server
                     Input:
                         filename    - filename to delete in remote folder
                     Return:
 in error case else 0
                     """
                     try:
                         self.ftp.delete(filename)
                     except:
                         print 'Error deleting remote file:%s'%filename
                         return 1
                     return 0
                 def download(self,filename,localfolder):
                     """
                     download is used to downloading file from remote folder into local folder
                     Inputs:
                         filename    - filename to donwload
                         localfolder    - directory local to store filename
                     Returns:
                         self.status    - 1 in error case else 0
                     """
                     self.status = 0
                     if not(filename in self.fileList):
                         print 'filename:%s not exists'%filename
                         self.status = 1
                         return self.status
                     newfilename = os.path.join(localfolder,filename)
                     self.file = open(newfilename, 'wb')
                     try:
                         print 'Download: ' + filename
                         self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
                         print 'Download Complete'
                     except ftplib.all_errors:
                         print 'Error Downloading ' + filename
                         self.status = 1
                         return self.status
                     self.file.close()
                     return self.status
                 def __handleDownload(self,block):
                     """
                     __handleDownload is used to handle writing file
                     """
                     self.file.write(block)
                 def upload(self,filename,remotefolder=None):
                     """
                     upload is used to uploading local file to remote directory
                     Inputs:
                         filename    - full path name of local file to store in remote directory
                         remotefolder    - remote directory
                     Returns:
                         self.status    - 1 in error case else 0
                     """
                     if remotefolder == None:
                         remotefolder = self.remotefolder
                     self.status = 0
                     try:
                         self.ftp.cwd(remotefolder)
                         self.file = open(filename, 'rb')
                         (head, tail) = os.path.split(filename)
                         command = "STOR " + tail
                         print 'Uploading: ' + tail
                         self.ftp.storbinary(command, self.file)
                         print 'Upload Completed'
                     except ftplib.all_errors:
                         print 'Error Uploading ' + tail
                         self.status = 1
                         return self.status
                     self.file.close()
                     #back to initial directory in __init__()
                     self.ftp.cwd(self.remotefolder)
                     return self.status
                 def dir(self,remotefolder):
                     """
                     dir is used to change working directory of remote server and get folder and file list
                     Input:
                         remotefolder    - current working directory
                     Affects:
                         self.fileList    - file list of working directory
                     Return:
                         infoList    - list with filenames and size of file in bytes
                         self.folderList    -    folder list
                     """
                     self.remotefolder = remotefolder
                     print 'Change to ' + self.remotefolder
                     try:
                         self.ftp.cwd(remotefolder)
                     except ftplib.all_errors:
                         print 'Error Change to ' + self.remotefolder
                         infoList = None
                         self.folderList = None
                         return infoList,self.folderList
                     self.dirList = []
                     try:
                         self.dirList = self.ftp.nlst()
                     except ftplib.error_perm, resp:
                         if str(resp) == "550 No files found":
                                 print "no files in this directory"
                                 infoList = None
                                 self.folderList = None
                                 return infoList,self.folderList
                     except ftplib.all_errors:
                         print 'Error Displaying Dir-Files'
                         infoList = None
                         self.folderList = None
                         return infoList,self.folderList
                     infoList = []
                     self.fileList = []
                     self.folderList = []
                     for f in self.dirList:
                         name,ext = os.path.splitext(f)
                         if ext != '':
                             self.fileList.append(f)
                             value = (f,self.ftp.size(f))
                             infoList.append(value)
                         if ext == '':
                             self.folderList.append(f)
                     return infoList,self.folderList
                 def close(self):
                     """
                     close is used to close and end FTP connection
                     Inputs: None
                     Return: void
                     """
                     self.ftp.close()
             class SendByFTP(Operation):
                 def __init__(self):
                     self.status = 1
                     self.counter = 0
                 def error_print(self, ValueError):
                     print ValueError, 'Error FTP'
                     print "don't worry the program is running..."
                 def worker_ftp(self, server, username, password, remotefolder, filenameList):
                     self.ftpClientObj = FTP(server, username, password, remotefolder)
                     for filename in filenameList:
                         self.ftpClientObj.upload(filename)
                     self.ftpClientObj.close()
                 def ftp_thread(self, server, username, password, remotefolder):
                     if not(self.status):
                         return
                     import multiprocessing
                     p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
                     p.start()
                     p.join(3)
                     if p.is_alive():
                         p.terminate()
                         p.join()
                         print 'killing ftp process...'
                         self.status = 0
                         return
                     self.status = 1
                     return
                 def filterByExt(self, ext, localfolder):
                     fnameList = glob.glob1(localfolder,ext)
                     self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
                     if len(self.filenameList) == 0:
                         self.status = 0
                 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
                     self.counter += 1
                     if self.counter >= period:
                         self.filterByExt(ext, localfolder)
                         self.ftp_thread(server, username, password, remotefolder)
                         self.counter = 0
                     self.status = 1

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages