schain Commit - r892:30b3788062ca · Jicamarca Repository

add multiSchain (@jchavez)

jespinoza -

r892:30b3788062ca

parent child

Context file:

r892:30b3788062ca

Collapse all files

schainpy/scripts/PPD.py created 644 +82 0

@@ -0,0 +1,82
	1	import argparse
	2
	3	from schainpy.controller import Project, multiSchain
	4
	5	desc = "HF_EXAMPLE"
	6
	7	def fiber(cursor, skip, q, dt):
	8
	9	controllerObj = Project()
	10
	11	controllerObj.setup(id='191', name='test01', description=desc)
	12
	13	readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
	14	path='/data/workspace/data/julia/',
	15	startDate=dt,
	16	endDate=dt,
	17	startTime="00:00:00",
	18	endTime="23:59:59",
	19	online=0,
	20	#set=1426485881,
	21	delay=10,
	22	walk=1,
	23	queue=q,
	24	cursor=cursor,
	25	skip=skip,
	26	#timezone=-5*3600
	27	)
	28
	29	# #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
	30	#
	31	procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
	32	# opObj11 = procUnitConfObj2.addParameter(name='pairsList', value='(0,1)', format='pairslist')
	33	#
	34	# procUnitConfObj2 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
	35
	36	# opObj11 = procUnitConfObj2.addOperation(name='SpectralMoments', optype='other')
	37
	38	#
	39	# opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
	40	# opObj11.addParameter(name='id', value='1000', format='int')
	41	# opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
	42	# opObj11.addParameter(name='channelList', value='0', format='intlist')
	43	# opObj11.addParameter(name='zmin', value='-120', format='float')
	44	# opObj11.addParameter(name='zmax', value='-70', format='float')
	45	# opObj11.addParameter(name='save', value='1', format='int')
	46	# opObj11.addParameter(name='figpath', value=figpath, format='str')
	47
	48	# opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
	49	# opObj11.addParameter(name='id', value='2000', format='int')
	50	# opObj11.addParameter(name='wintitle', value='HF_Jicamarca', format='str')
	51	# opObj11.addParameter(name='showprofile', value='0', format='int')
	52	# opObj11.addParameter(name='channelList', value='0', format='intlist')
	53	# # opObj11.addParameter(name='xmin', value='0', format='float')
	54	# opObj11.addParameter(name='xmin', value='0', format='float')
	55	# opObj11.addParameter(name='xmax', value='24', format='float')
	56	#
	57	# opObj11.addParameter(name='zmin', value='-110', format='float')
	58	# opObj11.addParameter(name='zmax', value='-70', format='float')
	59	# opObj11.addParameter(name='save', value='0', format='int')
	60	# opObj11.addParameter(name='figpath', value='/tmp/', format='str')
	61
	62	opObj12 = procUnitConfObj2.addOperation(name='PublishData', optype='other')
	63	opObj12.addParameter(name='zeromq', value=1, format='int')
	64
	65	# print "Escribiendo el archivo XML"
	66	# controllerObj.writeXml(filename)
	67	# print "Leyendo el archivo XML"
	68	# controllerObj.readXml(filename)
	69
	70	controllerObj.createObjects()
	71	controllerObj.connectObjects()
	72
	73	# timeit.timeit('controllerObj.run()', number=2)
	74
	75	controllerObj.run()
	76
	77
	78	if __name__ == '__main__':
	79	parser = argparse.ArgumentParser(description='Set number of parallel processes')
	80	parser.add_argument('--nProcess', default=2, type=int)
	81	args = parser.parse_args()
	82	multiSchain(fiber, nProcess=args.nProcess, startDate='2016/08/19', endDate='2016/08/20')

schainpy/controller.py +56 -3

             '''
             Created on September , 2012
             @author:
             '''
             import sys
             import ast
             import datetime
             import traceback
+            from multiprocessing import Process, Queue, cpu_count
             import schainpy
             import schainpy.admin
             from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
             from xml.dom import minidom
             from schainpy.model import *
             from time import sleep
             def prettify(elem):
                 """Return a pretty-printed XML string for the Element.
                 """
                 rough_string = tostring(elem, 'utf-8')
                 reparsed = minidom.parseString(rough_string)
                 return reparsed.toprettyxml(indent="  ")
+            def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None):
+                skip = 0
+                cursor = 0
+                nFiles = None
+                processes = []
+                dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
+                dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
+                days = (dt2 - dt1).days
+                print days
+                for day in range(days+1):
+                    skip = 0
+                    cursor = 0
+                    q = Queue()
+                    processes = []
+                    dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
+                    firstProcess = Process(target=child, args=(cursor, skip, q, dt))
+                    firstProcess.start()
+                    nFiles = q.get()
+                    firstProcess.terminate()
+                    skip = int(math.ceil(nFiles/nProcess))
+                    try:
+                        while True:
+                            processes.append(Process(target=child, args=(cursor, skip, q, dt)))
+                            processes[cursor].start()
+                            if nFiles < cursor*skip:
+                                break
+                            cursor += 1
+                    except KeyboardInterrupt:
+                        for process in processes:
+                            process.terminate()
+                            process.join()
+                    for process in processes:
+                        process.join()
+                        #process.terminate()
+                    sleep(3)
+                try:
+                    while True:
+                        pass
+                except KeyboardInterrupt:
+                    for process in processes:
+                        process.terminate()
+                        process.join()
             class ParameterConf():
                 id = None
                 name = None
                 value = None
                 format = None
                 __formated_value = None
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.format = 'str'
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getValue(self):
                     value = self.value
                     format = self.format
                     if self.__formated_value != None:
                         return self.__formated_value
+                    if format == 'obj':
+                        return value
                     if format == 'str':
                         self.__formated_value = str(value)
                         return self.__formated_value
                     if value == '':
                         raise ValueError, "%s: This parameter value is empty" %self.name
                     if format == 'list':
                         strList = value.split(',')
                         self.__formated_value = strList
                         return self.__formated_value
                     if format == 'intlist':
                         """
                         Example:
                             value = (0,1,2)
                         """
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             new_value = [int(new_value)]
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'floatlist':
                         """
                         Example:
                             value = (0.5, 1.4, 2.7)
                         """
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             new_value = [float(new_value)]
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'date':
                         strList = value.split('/')
                         intList = [int(x) for x in strList]
                         date = datetime.date(intList[0], intList[1], intList[2])
                         self.__formated_value = date
                         return self.__formated_value
                     if format == 'time':
                         strList = value.split(':')
                         intList = [int(x) for x in strList]
                         time = datetime.time(intList[0], intList[1], intList[2])
                         self.__formated_value = time
                         return self.__formated_value
                     if format == 'pairslist':
                         """
                         Example:
                             value = (0,1),(1,2)
                         """
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             raise ValueError, "%s has to be a tuple or list of pairs" %value
                         if type(new_value[0]) not in (tuple, list):
                             if len(new_value) != 2:
                                 raise ValueError, "%s has to be a tuple or list of pairs" %value
                             new_value = [new_value]
                         for thisPair in new_value:
                             if len(thisPair) != 2:
                                 raise ValueError, "%s has to be a tuple or list of pairs" %value
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'multilist':
                         """
                         Example:
                             value = (0,1,2),(3,4,5)
                         """
                         multiList = ast.literal_eval(value)
                         if type(multiList[0]) == int:
                             multiList = ast.literal_eval("(" + value + ")")
                         self.__formated_value = multiList
                         return self.__formated_value
                     if format == 'bool':
                         value = int(value)
                     if format == 'int':
                         value = float(value)
                     format_func = eval(format)
                     self.__formated_value = format_func(value)
                     return self.__formated_value
                 def updateId(self, new_id):
                     self.id = str(new_id)
                 def setup(self, id, name, value, format='str'):
                     self.id = str(id)
                     self.name = name
-                    self.value = str(value)
+                    if format == 'obj':
+                        self.value = value
+                    else:
+                        self.value = str(value)
                     self.format = str.lower(format)
                     self.getValue()
                     return 1
                 def update(self, name, value, format='str'):
                     self.name = name
                     self.value = str(value)
                     self.format = format
                 def makeXml(self, opElement):
                     parmElement = SubElement(opElement, self.ELEMENTNAME)
                     parmElement.set('id', str(self.id))
                     parmElement.set('name', self.name)
                     parmElement.set('value', self.value)
                     parmElement.set('format', self.format)
                 def readXml(self, parmElement):
                     self.id = parmElement.get('id')
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                     self.format = str.lower(parmElement.get('format'))
                     #Compatible with old signal chain version
                     if self.format == 'int' and self.name == 'idfigure':
                         self.name = 'id'
                 def printattr(self):
                     print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
             class OperationConf():
                 id = None
                 name = None
                 priority = None
                 type = None
                 parmConfObjList = []
                 ELEMENTNAME = 'Operation'
                 def __init__(self):
                     self.id = '0'
                     self.name = None
                     self.priority = None
                     self.type = 'self'
                 def __getNewId(self):
                     return int(self.id)*10 + len(self.parmConfObjList) + 1
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     n = 1
                     for parmObj in self.parmConfObjList:
                         idParm = str(int(new_id)*10 + n)
                         parmObj.updateId(idParm)
                         n += 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getParameterObjList(self):
                     return self.parmConfObjList
                 def getParameterObj(self, parameterName):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.name != parameterName:
                             continue
                         return parmConfObj
                     return None
                 def getParameterObjfromValue(self, parameterValue):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.getValue() != parameterValue:
                             continue
                         return parmConfObj.getValue()
                     return None
                 def getParameterValue(self, parameterName):
                     parameterObj = self.getParameterObj(parameterName)
             #         if not parameterObj:
             #             return None
                     value = parameterObj.getValue()
                     return value
                 def getKwargs(self):
                     kwargs = {}
                     for parmConfObj in self.parmConfObjList:
                         if self.name == 'run' and parmConfObj.name == 'datatype':
                             continue
                         kwargs[parmConfObj.name] = parmConfObj.getValue()
                     return kwargs
                 def setup(self, id, name, priority, type):
                     self.id = str(id)
                     self.name = name
                     self.type = type
                     self.priority = priority
                     self.parmConfObjList = []
                 def removeParameters(self):
                     for obj in self.parmConfObjList:
                         del obj
                     self.parmConfObjList = []
                 def addParameter(self, name, value, format='str'):
                     id = self.__getNewId()
                     parmConfObj = ParameterConf()
                     if not parmConfObj.setup(id, name, value, format):
                         return None
                     self.parmConfObjList.append(parmConfObj)
                     return parmConfObj
                 def changeParameter(self, name, value, format='str'):
                     parmConfObj = self.getParameterObj(name)
                     parmConfObj.update(name, value, format)
                     return parmConfObj
                 def makeXml(self, procUnitElement):
                     opElement = SubElement(procUnitElement, self.ELEMENTNAME)
                     opElement.set('id', str(self.id))
                     opElement.set('name', self.name)
                     opElement.set('type', self.type)
                     opElement.set('priority', str(self.priority))
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.makeXml(opElement)
                 def readXml(self, opElement):
                     self.id = opElement.get('id')
                     self.name = opElement.get('name')
                     self.type = opElement.get('type')
                     self.priority = opElement.get('priority')
                     #Compatible with old signal chain version
                     #Use of 'run' method instead 'init'
                     if self.type == 'self' and self.name == 'init':
                         self.name = 'run'
                     self.parmConfObjList = []
                     parmElementList = opElement.iter(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         #Compatible with old signal chain version
                         #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
                         if self.type != 'self' and self.name == 'Plot':
                             if parmConfObj.format == 'str' and parmConfObj.name == 'type':
                                 self.name = parmConfObj.value
                                 continue
                         self.parmConfObjList.append(parmConfObj)
                 def printattr(self):
                     print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
                                                                           self.id,
                                                                           self.name,
                                                                           self.type,
                                                                           self.priority)
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.printattr()
                 def createObject(self, plotter_queue=None):
                     if self.type == 'self':
                         raise ValueError, "This operation type cannot be created"
                     if self.type == 'plotter':
                         #Plotter(plotter_name)
                         if not plotter_queue:
                             raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
                         opObj = Plotter(self.name, plotter_queue)
                     if self.type == 'external' or self.type == 'other':
                         className = eval(self.name)
                         kwargs = self.getKwargs()
                         opObj = className(**kwargs)
                     return opObj
             class ProcUnitConf():
                 id = None
                 name = None
                 datatype = None
                 inputId = None
                 parentId = None
                 opConfObjList = []
                 procUnitObj = None
                 opObjList = []
                 ELEMENTNAME = 'ProcUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.opConfObjList = []
                     self.procUnitObj = None
                     self.opObjDict = {}
                 def __getPriority(self):
                     return len(self.opConfObjList)+1
                 def __getNewId(self):
                     return int(self.id)*10 + len(self.opConfObjList) + 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def updateId(self, new_id, parentId=parentId):
                     new_id = int(parentId)*10 + (int(self.id) % 10)
                     new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
                     #If this proc unit has not inputs
                     if self.inputId == '0':
                         new_inputId = 0
                     n = 1
                     for opConfObj in self.opConfObjList:
                         idOp = str(int(new_id)*10 + n)
                         opConfObj.updateId(idOp)
                         n += 1
                     self.parentId = str(parentId)
                     self.id = str(new_id)
                     self.inputId = str(new_inputId)
                 def getInputId(self):
                     return self.inputId
                 def getOperationObjList(self):
                     return self.opConfObjList
                 def getOperationObj(self, name=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.name != name:
                             continue
                         return opConfObj
                     return None
                 def getOpObjfromParamValue(self, value=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
                             continue
                         return opConfObj
                     return None
                 def getProcUnitObj(self):
                     return self.procUnitObj
                 def setup(self, id, name, datatype, inputId, parentId=None):
                     #Compatible with old signal chain version
                     if datatype==None and name==None:
                         raise ValueError, "datatype or name should be defined"
                     if name==None:
                         if 'Proc' in datatype:
                             name = datatype
                         else:
                             name = '%sProc' %(datatype)
                     if datatype==None:
                         datatype = name.replace('Proc','')
                     self.id = str(id)
                     self.name = name
                     self.datatype = datatype
                     self.inputId = inputId
                     self.parentId = parentId
                     self.opConfObjList = []
                     self.addOperation(name='run', optype='self')
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                     self.addOperation(name='run')
                 def addParameter(self, **kwargs):
                     '''
                     Add parameters to "run" operation
                     '''
                     opObj = self.opConfObjList[0]
                     opObj.addParameter(**kwargs)
                     return opObj
                 def addOperation(self, name, optype='self'):
                     id = self.__getNewId()
                     priority = self.__getPriority()
                     opConfObj = OperationConf()
                     opConfObj.setup(id, name=name, priority=priority, type=optype)
                     self.opConfObjList.append(opConfObj)
                     return opConfObj
                 def makeXml(self, projectElement):
                     procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
                     procUnitElement.set('id', str(self.id))
                     procUnitElement.set('name', self.name)
                     procUnitElement.set('datatype', self.datatype)
                     procUnitElement.set('inputId', str(self.inputId))
                     for opConfObj in self.opConfObjList:
                         opConfObj.makeXml(procUnitElement)
                 def readXml(self, upElement):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.inputId = upElement.get('inputId')
                     if self.ELEMENTNAME == "ReadUnit":
                         self.datatype = self.datatype.replace("Reader", "")
                     if self.ELEMENTNAME == "ProcUnit":
                         self.datatype = self.datatype.replace("Proc", "")
                     if self.inputId == 'None':
                         self.inputId = '0'
                     self.opConfObjList = []
                     opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement)
                         self.opConfObjList.append(opConfObj)
                 def printattr(self):
                     print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
                                                                          self.id,
                                                                          self.name,
                                                                          self.datatype,
                                                                          self.inputId)
                     for opConfObj in self.opConfObjList:
                         opConfObj.printattr()
                 def getKwargs(self):
                     opObj = self.opConfObjList[0]
                     kwargs = opObj.getKwargs()
                     return kwargs
                 def createObjects(self, plotter_queue=None):
                     className = eval(self.name)
                     kwargs = self.getKwargs()
                     procUnitObj = className(**kwargs)
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self':
                             continue
                         opObj = opConfObj.createObject(plotter_queue)
                         self.opObjDict[opConfObj.id] = opObj
                         procUnitObj.addOperation(opObj, opConfObj.id)
                     self.procUnitObj = procUnitObj
                     return procUnitObj
                 def run(self):
                     is_ok = False
                     for opConfObj in self.opConfObjList:
                         kwargs = {}
                         for parmConfObj in opConfObj.getParameterObjList():
                             if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
                                 continue
                             kwargs[parmConfObj.name] = parmConfObj.getValue()
                         #ini = time.time()
                         #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
                         sts = self.procUnitObj.call(opType = opConfObj.type,
                                                     opName = opConfObj.name,
                                                     opId = opConfObj.id,
                                                     )
             #             total_time = time.time() - ini
             #
             #             if total_time > 0.002:
             #                 print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
                         is_ok = is_ok or sts
                     return is_ok
                 def close(self):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self':
                             continue
                         opObj = self.procUnitObj.getOperationObj(opConfObj.id)
                         opObj.close()
                     self.procUnitObj.close()
                     return
             class ReadUnitConf(ProcUnitConf):
                 path = None
                 startDate = None
                 endDate = None
                 startTime = None
                 endTime = None
                 ELEMENTNAME = 'ReadUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.parentId = None
                     self.opConfObjList = []
                     self.opObjList = []
                 def getElementName(self):
                     return self.ELEMENTNAME
-                def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, **kwargs):
+                def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, queue=None, **kwargs):
                     #Compatible with old signal chain version
                     if datatype==None and name==None:
                         raise ValueError, "datatype or name should be defined"
                     if name==None:
                         if 'Reader' in datatype:
                             name = datatype
                         else:
                             name = '%sReader' %(datatype)
                     if datatype==None:
                         datatype = name.replace('Reader','')
                     self.id = id
                     self.name = name
                     self.datatype = datatype
                     self.path = os.path.abspath(path)
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.inputId = '0'
                     self.parentId = parentId
+                    self.queue = queue
                     self.addRunOperation(**kwargs)
                 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
                     #Compatible with old signal chain version
                     if datatype==None and name==None:
                         raise ValueError, "datatype or name should be defined"
                     if name==None:
                         if 'Reader' in datatype:
                             name = datatype
                         else:
                             name = '%sReader' %(datatype)
                     if datatype==None:
                         datatype = name.replace('Reader','')
                     self.datatype = datatype
                     self.name = name
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.inputId = '0'
                     self.parentId = parentId
                     self.updateRunOperation(**kwargs)
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                 def addRunOperation(self, **kwargs):
                     opObj = self.addOperation(name = 'run', optype = 'self')
                     opObj.addParameter(name='datatype' , value=self.datatype, format='str')
                     opObj.addParameter(name='path'     , value=self.path, format='str')
                     opObj.addParameter(name='startDate' , value=self.startDate, format='date')
                     opObj.addParameter(name='endDate'   , value=self.endDate, format='date')
                     opObj.addParameter(name='startTime' , value=self.startTime, format='time')
                     opObj.addParameter(name='endTime'   , value=self.endTime, format='time')
                     for key, value in kwargs.items():
                         opObj.addParameter(name=key, value=value, format=type(value).__name__)
                     return opObj
                 def updateRunOperation(self, **kwargs):
                     opObj = self.getOperationObj(name = 'run')
                     opObj.removeParameters()
                     opObj.addParameter(name='datatype' , value=self.datatype, format='str')
                     opObj.addParameter(name='path'     , value=self.path, format='str')
                     opObj.addParameter(name='startDate' , value=self.startDate, format='date')
                     opObj.addParameter(name='endDate'   , value=self.endDate, format='date')
                     opObj.addParameter(name='startTime' , value=self.startTime, format='time')
                     opObj.addParameter(name='endTime'   , value=self.endTime, format='time')
                     for key, value in kwargs.items():
                         opObj.addParameter(name=key, value=value, format=type(value).__name__)
                     return opObj
             #     def makeXml(self, projectElement):
             #
             #         procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
             #         procUnitElement.set('id', str(self.id))
             #         procUnitElement.set('name', self.name)
             #         procUnitElement.set('datatype', self.datatype)
             #         procUnitElement.set('inputId', str(self.inputId))
             #
             #         for opConfObj in self.opConfObjList:
             #             opConfObj.makeXml(procUnitElement)
                 def readXml(self, upElement):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.inputId = upElement.get('inputId')
                     if self.ELEMENTNAME == "ReadUnit":
                         self.datatype = self.datatype.replace("Reader", "")
                     if self.inputId == 'None':
                         self.inputId = '0'
                     self.opConfObjList = []
                     opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement)
                         self.opConfObjList.append(opConfObj)
                         if opConfObj.name == 'run':
                             self.path = opConfObj.getParameterValue('path')
                             self.startDate = opConfObj.getParameterValue('startDate')
                             self.endDate = opConfObj.getParameterValue('endDate')
                             self.startTime = opConfObj.getParameterValue('startTime')
                             self.endTime = opConfObj.getParameterValue('endTime')
             class Project():
                 id = None
                 name = None
                 description = None
                 filename = None
                 procUnitConfObjDict = None
                 ELEMENTNAME = 'Project'
                 plotterQueue = None
                 def __init__(self, plotter_queue=None):
                     self.id = None
                     self.name = None
                     self.description = None
                     self.plotterQueue = plotter_queue
                     self.procUnitConfObjDict = {}
                 def __getNewId(self):
                     idList = self.procUnitConfObjDict.keys()
                     id = int(self.id)*10
                     while True:
                         id += 1
                         if str(id) in idList:
                             continue
                         break
                     return str(id)
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     keyList = self.procUnitConfObjDict.keys()
                     keyList.sort()
                     n = 1
                     newProcUnitConfObjDict = {}
                     for procKey in keyList:
                         procUnitConfObj = self.procUnitConfObjDict[procKey]
                         idProcUnit = str(int(self.id)*10 + n)
                         procUnitConfObj.updateId(idProcUnit, parentId = self.id)
                         newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
                         n += 1
                     self.procUnitConfObjDict = newProcUnitConfObjDict
                 def setup(self, id, name, description):
                     self.id = str(id)
                     self.name = name
                     self.description = description
                 def update(self, name, description):
                     self.name = name
                     self.description = description
                 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
                     if id is None:
                         idReadUnit = self.__getNewId()
                     else:
                         idReadUnit = str(id)
                     readUnitConfObj = ReadUnitConf()
                     readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
                     self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     return readUnitConfObj
                 def addProcUnit(self, inputId='0', datatype=None, name=None):
                     idProcUnit = self.__getNewId()
                     procUnitConfObj = ProcUnitConf()
                     procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
                     self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     return procUnitConfObj
                 def removeProcUnit(self, id):
                     if id in self.procUnitConfObjDict.keys():
                         self.procUnitConfObjDict.pop(id)
                 def getReadUnitId(self):
                     readUnitConfObj = self.getReadUnitObj()
                     return readUnitConfObj.id
                 def getReadUnitObj(self):
                     for obj in self.procUnitConfObjDict.values():
                         if obj.getElementName() == "ReadUnit":
                             return obj
                     return None
                 def getProcUnitObj(self, id=None, name=None):
                     if id != None:
                         return self.procUnitConfObjDict[id]
                     if name != None:
                         return self.getProcUnitObjByName(name)
                     return None
                 def getProcUnitObjByName(self, name):
                     for obj in self.procUnitConfObjDict.values():
                         if obj.name == name:
                             return obj
                     return None
                 def procUnitItems(self):
                     return self.procUnitConfObjDict.items()
                 def makeXml(self):
                     projectElement = Element('Project')
                     projectElement.set('id', str(self.id))
                     projectElement.set('name', self.name)
                     projectElement.set('description', self.description)
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.makeXml(projectElement)
                     self.projectElement = projectElement
                 def writeXml(self, filename=None):
                     if filename == None:
                         if self.filename:
                             filename = self.filename
                         else:
                             filename = "schain.xml"
                     if not filename:
                         print "filename has not been defined. Use setFilename(filename) for do it."
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.access(os.path.dirname(abs_file), os.W_OK):
                         print "No write permission on %s" %os.path.dirname(abs_file)
                         return 0
                     if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
                         print "File %s already exists and it could not be overwriten" %abs_file
                         return 0
                     self.makeXml()
                     ElementTree(self.projectElement).write(abs_file, method='xml')
                     self.filename = abs_file
                     return 1
                 def readXml(self, filename = None):
                     if not filename:
                         print "filename is not defined"
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.path.isfile(abs_file):
                         print "%s file does not exist" %abs_file
                         return 0
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     try:
                         self.projectElement = ElementTree().parse(abs_file)
                     except:
                         print "Error reading %s, verify file format" %filename
                         return 0
                     self.project = self.projectElement.tag
                     self.id = self.projectElement.get('id')
                     self.name = self.projectElement.get('name')
                     self.description = self.projectElement.get('description')
                     readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
                     for readUnitElement in readUnitElementList:
                         readUnitConfObj = ReadUnitConf()
                         readUnitConfObj.readXml(readUnitElement)
                         if readUnitConfObj.parentId == None:
                             readUnitConfObj.parentId = self.id
                         self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
                     for procUnitElement in procUnitElementList:
                         procUnitConfObj = ProcUnitConf()
                         procUnitConfObj.readXml(procUnitElement)
                         if procUnitConfObj.parentId == None:
                             procUnitConfObj.parentId = self.id
                         self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     self.filename = abs_file
                     return 1
                 def printattr(self):
                     print "Project[%s]: name = %s, description = %s" %(self.id,
                                                                           self.name,
                                                                           self.description)
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.printattr()
                 def createObjects(self):
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.createObjects(self.plotterQueue)
                 def __connect(self, objIN, thisObj):
                     thisObj.setInput(objIN.getOutputObj())
                 def connectObjects(self):
                     for thisPUConfObj in self.procUnitConfObjDict.values():
                         inputId = thisPUConfObj.getInputId()
                         if int(inputId) == 0:
                             continue
                         #Get input object
                         puConfINObj = self.procUnitConfObjDict[inputId]
                         puObjIN = puConfINObj.getProcUnitObj()
                         #Get current object
                         thisPUObj = thisPUConfObj.getProcUnitObj()
                         self.__connect(puObjIN, thisPUObj)
                 def __handleError(self, procUnitConfObj, send_email=True):
                     import socket
                     err = traceback.format_exception(sys.exc_info()[0],
                                                      sys.exc_info()[1],
                                                      sys.exc_info()[2])
                     print "***** Error occurred in %s *****" %(procUnitConfObj.name)
                     print "***** %s" %err[-1]
                     message = "".join(err)
                     sys.stderr.write(message)
                     if not send_email:
                         return
                     subject =  "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
                     subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
                     subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
                     subtitle += "Working directory: %s\n" %os.path.abspath("./")
                     subtitle += "Configuration file: %s\n" %self.filename
                     subtitle += "Time: %s\n" %str(datetime.datetime.now())
                     readUnitConfObj = self.getReadUnitObj()
                     if readUnitConfObj:
                         subtitle += "\nInput parameters:\n"
                         subtitle += "[Data path = %s]\n" %readUnitConfObj.path
                         subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
                         subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
                         subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
                         subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
                         subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
                     adminObj = schainpy.admin.SchainNotify()
                     adminObj.sendAlert(message=message,
                                        subject=subject,
                                        subtitle=subtitle,
                                        filename=self.filename)
                 def isPaused(self):
                     return 0
                 def isStopped(self):
                     return 0
                 def runController(self):
                     """
                     returns 0 when this process has been stopped, 1 otherwise
                     """
                     if self.isPaused():
                         print "Process suspended"
                         while True:
                             sleep(0.1)
                             if not self.isPaused():
                                 break
                             if self.isStopped():
                                 break
                         print "Process reinitialized"
                     if self.isStopped():
                         print "Process stopped"
                         return 0
                     return 1
                 def setFilename(self, filename):
                     self.filename = filename
                 def setPlotterQueue(self, plotter_queue):
                     raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
                 def getPlotterQueue(self):
                     raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
                 def useExternalPlotter(self):
                     raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
                 def run(self):
                     print
                     print "*"*60
                     print "   Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
                     print "*"*60
                     print
                     keyList = self.procUnitConfObjDict.keys()
                     keyList.sort()
                     while(True):
                         is_ok = False
                         for procKey in keyList:
             #                 print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
                             procUnitConfObj = self.procUnitConfObjDict[procKey]
                             try:
                                 sts = procUnitConfObj.run()
                                 is_ok = is_ok or sts
                             except KeyboardInterrupt:
                                 is_ok = False
                                 break
                             except ValueError, e:
                                 sleep(0.5)
                                 self.__handleError(procUnitConfObj, send_email=True)
                                 is_ok = False
                                 break
                             except:
                                 sleep(0.5)
                                 self.__handleError(procUnitConfObj)
                                 is_ok = False
                                 break
                         #If every process unit finished so end process
                         if not(is_ok):
             #                 print "Every process unit have finished"
                             break
                         if not self.runController():
                             break
                     #Closing every process
                     for procKey in keyList:
                         procUnitConfObj = self.procUnitConfObjDict[procKey]
                         procUnitConfObj.close()
                     print "Process finished"
                 def start(self):
                     self.writeXml()
                     self.createObjects()
                     self.connectObjects()
                     self.run()

schainpy/model/graphics/jroplot_data.py +7 -6

             import os
             import zmq
             import time
             import numpy
             import datetime
             import numpy as np
             import matplotlib.pyplot as plt
             from mpl_toolkits.axes_grid1 import make_axes_locatable
             from matplotlib.ticker import FuncFormatter, LinearLocator
             from multiprocessing import Process
             from schainpy.model.proc.jroproc_base import Operation
             #plt.ion()
             func = lambda x, pos: ('%s') %(datetime.datetime.utcfromtimestamp(x).strftime('%H:%M'))
             d1970 = datetime.datetime(1970,1,1)
             class PlotData(Operation, Process):
                 CODE = 'Figure'
                 colormap = 'jet'
                 __MAXNUMX = 80
                 __MAXNUMY = 80
                 __missing = 1E30
                 def __init__(self, **kwargs):
                     Operation.__init__(self)
                     Process.__init__(self)
                     self.mp = False
                     self.dataOut = None
                     self.isConfig = False
                     self.figure = None
                     self.axes = []
                     self.localtime = kwargs.pop('localtime', True)
                     self.show = kwargs.get('show', True)
                     self.save = kwargs.get('save', False)
                     self.colormap = kwargs.get('colormap', self.colormap)
                     self.showprofile = kwargs.get('showprofile', False)
                     self.title = kwargs.get('wintitle', '')
                     self.xaxis = kwargs.get('xaxis', 'time')
                     self.zmin = kwargs.get('zmin', None)
                     self.zmax = kwargs.get('zmax', None)
                     self.xmin = kwargs.get('xmin', None)
                     self.xmax = kwargs.get('xmax', None)
                     self.xrange = kwargs.get('xrange', 24)
                     self.ymin = kwargs.get('ymin', None)
                     self.ymax = kwargs.get('ymax', None)
                 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
                     if x_buffer.shape[0] < 2:
                         return x_buffer, y_buffer, z_buffer
                     deltas = x_buffer[1:] - x_buffer[0:-1]
                     x_median = np.median(deltas)
                     index = np.where(deltas > 5*x_median)
                     if len(index[0]) != 0:
                         z_buffer[::,index[0],::] = self.__missing
                         z_buffer = np.ma.masked_inside(z_buffer,
 .99*self.__missing,
 .01*self.__missing)
                     return x_buffer, y_buffer, z_buffer
                 def decimate(self):
                     dx = int(len(self.x)/self.__MAXNUMX) + 1
                     dy = int(len(self.y)/self.__MAXNUMY) + 1
                     x = self.x[::dx]
                     y = self.y[::dy]
                     z = self.z[::, ::dx, ::dy]
                     return x, y, z
                 def __plot(self):
                     print 'plotting...{}'.format(self.CODE)
                     self.plot()
-                    self.figure.suptitle('{} {}'.format(self.title, self.CODE.upper()))
+                    self.figure.suptitle('{} {} - Date:{}'.format(self.title, self.CODE.upper(),
+                                                                  datetime.datetime.utcfromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')))
                     if self.save:
                         figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,
                                                                              datetime.datetime.utcfromtimestamp(self.times[-1]).strftime('%y%m%d_%H%M%S')))
                         print 'Saving figure: {}'.format(figname)
                         self.figure.savefig(figname)
                     self.figure.canvas.draw()
                 def plot(self):
                     print 'plotting...{}'.format(self.CODE.upper())
                     return
                 def run(self):
                     print '[Starting] {}'.format(self.name)
                     context = zmq.Context()
                     receiver = context.socket(zmq.SUB)
                     receiver.setsockopt(zmq.SUBSCRIBE, '')
                     receiver.setsockopt(zmq.CONFLATE, True)
                     receiver.connect("ipc:///tmp/zmq.plots")
                     while True:
                         try:
                         #if True:
                             self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
                             self.dataOut = self.data['dataOut']
                             self.times = self.data['times']
                             self.times.sort()
                             self.min_time = self.times[0]
                             self.max_time = self.times[-1]
                             if self.isConfig is False:
                                 self.setup()
                                 self.isConfig = True
                             self.__plot()
                             if 'ENDED' in self.data:
                                 #self.setup()
                                 #self.__plot()
                                 pass
                         except zmq.Again as e:
                             print 'Waiting for data...'
                             plt.pause(5)
                             #time.sleep(3)
                 def close(self):
                     if self.dataOut:
                         self._plot()
             class PlotSpectraData(PlotData):
                 CODE = 'spc'
                 colormap = 'jro'
                 def setup(self):
                     ncolspan = 1
                     colspan = 1
                     self.ncols = int(numpy.sqrt(self.dataOut.nChannels)+0.9)
                     self.nrows = int(self.dataOut.nChannels*1./self.ncols + 0.9)
                     self.width = 3.6*self.ncols
                     self.height = 3.2*self.nrows
                     if self.showprofile:
                         ncolspan = 3
                         colspan = 2
                         self.width += 1.2*self.ncols
                     self.ylabel = 'Range [Km]'
                     self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
                     if self.figure is None:
                         self.figure = plt.figure(figsize=(self.width, self.height),
                                                  edgecolor='k',
                                                  facecolor='w')
                     else:
                         self.figure.clf()
                     n = 0
                     for y in range(self.nrows):
                         for x in range(self.ncols):
                             if n>=self.dataOut.nChannels:
                                 break
                             ax = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan), 1, colspan)
                             if self.showprofile:
                                 ax.ax_profile = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan+colspan), 1, 1)
                             ax.firsttime = True
                             self.axes.append(ax)
                             n += 1
                     self.figure.subplots_adjust(wspace=0.9, hspace=0.5)
                     self.figure.show()
                 def plot(self):
                     if self.xaxis == "frequency":
                         x = self.dataOut.getFreqRange(1)/1000.
                         xlabel = "Frequency (kHz)"
                     elif self.xaxis == "time":
                         x = self.dataOut.getAcfRange(1)
                         xlabel = "Time (ms)"
                     else:
                         x = self.dataOut.getVelRange(1)
                         xlabel = "Velocity (m/s)"
                     y = self.dataOut.getHeiRange()
                     z = self.data[self.CODE]
                     for n, ax in enumerate(self.axes):
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else np.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.ymin = self.ymin if self.ymin else np.nanmin(y)
                             self.ymax = self.ymax if self.ymax else np.nanmax(y)
                             self.zmin = self.zmin if self.zmin else np.nanmin(z)
                             self.zmax = self.zmax if self.zmax else np.nanmax(z)
                             ax.plot = ax.pcolormesh(x, y, z[n].T,
                                                     vmin=self.zmin,
                                                     vmax=self.zmax,
                                                     cmap=plt.get_cmap(self.colormap)
                                                    )
                             divider = make_axes_locatable(ax)
                             cax = divider.new_horizontal(size='3%', pad=0.05)
                             self.figure.add_axes(cax)
                             plt.colorbar(ax.plot, cax)
                             ax.set_xlim(self.xmin, self.xmax)
                             ax.set_ylim(self.ymin, self.ymax)
                             ax.xaxis.set_major_locator(LinearLocator(5))
                             #ax.yaxis.set_major_locator(LinearLocator(4))
                             ax.set_ylabel(self.ylabel)
                             ax.set_xlabel(xlabel)
                             ax.firsttime = False
                             if self.showprofile:
                                 ax.plot_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
                                 ax.ax_profile.set_xlim(self.zmin, self.zmax)
                                 ax.ax_profile.set_ylim(self.ymin, self.ymax)
                                 ax.ax_profile.set_xlabel('dB')
                                 ax.ax_profile.grid(b=True, axis='x')
+                                ax.plot_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
+                                                                   color="k", linestyle="dashed", lw=2)[0]
                                 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
-                                noise = 10*numpy.log10(self.data['rti'][self.max_time][n]/self.dataOut.normFactor)
-                                ax.ax_profile.vlines(noise, self.ymin, self.ymax, colors="k", linestyle="dashed", lw=2)
                         else:
                             ax.plot.set_array(z[n].T.ravel())
-                            ax.set_title('{} {}'.format(self.titles[n],
-                                                        datetime.datetime.utcfromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
-                                         size=8)
                             if self.showprofile:
                                 ax.plot_profile.set_data(self.data['rti'][self.max_time][n], y)
+                                ax.plot_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
+                        ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
+                                     size=8)
             class PlotRTIData(PlotData):
                 CODE = 'rti'
                 colormap = 'jro'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = self.dataOut.nChannels
                     self.width = 10
                     self.height = 2.2*self.nrows
                     self.ylabel = 'Range [Km]'
                     self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
                     if self.figure is None:
                         self.figure = plt.figure(figsize=(self.width, self.height),
                                                  edgecolor='k',
                                                  facecolor='w')
                     else:
                         self.figure.clf()
                     for n in range(self.nrows):
                         ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
                         ax.firsttime = True
                         self.axes.append(ax)
                     self.figure.subplots_adjust(hspace=0.5)
                     self.figure.show()
                 def plot(self):
                     self.x = np.array(self.times)
                     self.y = self.dataOut.getHeiRange()
                     self.z = []
                     for ch in range(self.nrows):
                         self.z.append([self.data[self.CODE][t][ch] for t in self.times])
                     self.z = np.array(self.z)
                     for n, ax in enumerate(self.axes):
                         x, y, z = self.fill_gaps(*self.decimate())
                         if ax.firsttime:
                             self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
                             self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
                             self.zmin = self.zmin if self.zmin else np.nanmin(self.z)
                             zmax = self.zmax if self.zmax else np.nanmax(self.z)
                             plot = ax.pcolormesh(x, y, z[n].T,
                                                  vmin=self.zmin,
                                                  vmax=self.zmax,
                                                  cmap=plt.get_cmap(self.colormap)
                                                 )
                             divider = make_axes_locatable(ax)
                             cax = divider.new_horizontal(size='2%', pad=0.05)
                             self.figure.add_axes(cax)
                             plt.colorbar(plot, cax)
                             ax.set_ylim(self.ymin, self.ymax)
                             if self.xaxis=='time':
                                 ax.xaxis.set_major_formatter(FuncFormatter(func))
                                 ax.xaxis.set_major_locator(LinearLocator(6))
                             ax.yaxis.set_major_locator(LinearLocator(4))
                             ax.set_ylabel(self.ylabel)
                             if self.xmin is None:
                                 print 'is none'
                                 xmin = self.min_time
                             else:
                                 xmin = (datetime.datetime.combine(self.dataOut.datatime.date(),
                                                                  datetime.time(self.xmin, 0, 0))-d1970).total_seconds()
                             xmax = xmin+self.xrange*60*60
                             ax.set_xlim(xmin, xmax)
                             ax.firsttime = False
                         else:
                             ax.collections.remove(ax.collections[0])
                             plot = ax.pcolormesh(x, y, z[n].T,
                                                  vmin=self.zmin,
                                                  vmax=self.zmax,
                                                  cmap=plt.get_cmap(self.colormap)
                                                 )
                             ax.set_title('{} {}'.format(self.titles[n],
                                                         datetime.datetime.utcfromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
                                          size=8)
             class PlotCOHData(PlotRTIData):
                 CODE = 'coh'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = self.dataOut.nPairs
                     self.width = 10
                     self.height = 2.2*self.nrows
                     self.ylabel = 'Range [Km]'
                     self.titles = ['Channels {}'.format(x) for x in self.dataOut.pairsList]
                     if self.figure is None:
                         self.figure = plt.figure(figsize=(self.width, self.height),
                                                  edgecolor='k',
                                                  facecolor='w')
                     else:
                         self.figure.clf()
                     for n in range(self.nrows):
                         ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
                         ax.firsttime = True
                         self.axes.append(ax)
                     self.figure.subplots_adjust(hspace=0.5)
                     self.figure.show()
             class PlotSNRData(PlotRTIData):
                 CODE = 'coh'
             class PlotPHASEData(PlotCOHData):
                 CODE = 'phase'
                 colormap = 'seismic'

schainpy/model/io/jroIO_base.py +493 -473

             '''
             Created on Jul 2, 2014
             @author: roj-idl71
             '''
             import os
             import sys
             import glob
             import time
             import numpy
             import fnmatch
             import time, datetime
             #import h5py
             import traceback
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
             LOCALTIME = True
             def isNumber(cad):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float( cad )
                     return True
                 except:
                     return False
             def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
                 """
                 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                     endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 try:
                     fp = open(filename,'rb')
                 except IOError:
                     print "The file %s can't be opened" %(filename)
                     return 0
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return 0
                 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                     return 0
                 return 1
             def isTimeInRange(thisTime, startTime, endTime):
                 if endTime >= startTime:
                     if (thisTime < startTime) or (thisTime > endTime):
                         return 0
                     return 1
                 else:
                     if (thisTime < startTime) and (thisTime > endTime):
                         return 0
                     return 1
             def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                     startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                     endTime            :    tiempo final del rango seleccionado en formato datetime.time
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 try:
                     fp = open(filename,'rb')
                 except IOError:
                     print "The file %s can't be opened" %(filename)
                     return None
                 firstBasicHeaderObj = BasicHeader(LOCALTIME)
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
                 processingHeaderObj = ProcessingHeader()
                 lastBasicHeaderObj = BasicHeader(LOCALTIME)
                 sts = firstBasicHeaderObj.read(fp)
                 if not(sts):
                     print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
                     return None
                 if not systemHeaderObj.read(fp):
                     return None
                 if not radarControllerHeaderObj.read(fp):
                     return None
                 if not processingHeaderObj.read(fp):
                     return None
                 filesize = os.path.getsize(filename)
                 offset = processingHeaderObj.blockSize + 24 #header size
                 if filesize <= offset:
                     print "[Reading] %s: This file has not enough data" %filename
                     return None
                 fp.seek(-offset, 2)
                 sts = lastBasicHeaderObj.read(fp)
                 fp.close()
                 thisDatetime = lastBasicHeaderObj.datatime
                 thisTime_last_block = thisDatetime.time()
                 thisDatetime = firstBasicHeaderObj.datatime
                 thisDate = thisDatetime.date()
                 thisTime_first_block = thisDatetime.time()
                 #General case
                 #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                 #-----------o----------------------------o-----------
                 #       startTime                     endTime
                 if endTime >= startTime:
                     if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
                         return None
                     return thisDatetime
                 #If endTime < startTime then endTime belongs to the next day
                 #<<<<<<<<<<<o                            o>>>>>>>>>>>
                 #-----------o----------------------------o-----------
                 #        endTime                    startTime
                 if (thisDate == startDate) and (thisTime_last_block < startTime):
                     return None
                 if (thisDate == endDate) and (thisTime_first_block > endTime):
                     return None
                 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
                     return None
                 return thisDatetime
             def isFolderInDateRange(folder, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     folder            :    nombre completo del directorio.
                                            Su formato deberia ser "/path_root/?YYYYDDD"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el directorio no tiene el formato adecuado
                 """
                 basename = os.path.basename(folder)
                 if not isRadarFolder(basename):
                     print "The folder %s has not the rigth format" %folder
                     return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFolder(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def isFileInDateRange(filename, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                                            Su formato deberia ser "?YYYYDDDsss"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                                                 sss     :    set
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no tiene el formato adecuado
                 """
                 basename = os.path.basename(filename)
                 if not isRadarFile(basename):
                     print "The filename %s has not the rigth format" %filename
                     return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFile(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def getFileFromSet(path, ext, set):
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     try:
                         year = int(thisFile[1:5])
                         doy  = int(thisFile[5:8])
                     except:
                         continue
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
                 if len(myfile)!= 0:
                     return myfile[0]
                 else:
                     filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
                     print 'the filename %s does not exist'%filename
                     print '...going to the last file: '
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def getlastFileFromPath(path, ext):
                 """
                 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     year = thisFile[1:5]
                     if not isNumber(year):
                         continue
                     doy = thisFile[5:8]
                     if not isNumber(doy):
                         continue
                     year = int(year)
                     doy = int(doy)
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def checkForRealPath(path, foldercounter, year, doy, set, ext):
                 """
                 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
                 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
                 el path exacto de un determinado file.
                 Example    :
                     nombre correcto del file es  .../.../D2009307/P2009307367.ext
                     Entonces la funcion prueba con las siguientes combinaciones
                         .../.../y2009307367.ext
                         .../.../Y2009307367.ext
                         .../.../x2009307/y2009307367.ext
                         .../.../x2009307/Y2009307367.ext
                         .../.../X2009307/y2009307367.ext
                         .../.../X2009307/Y2009307367.ext
                     siendo para este caso, la ultima combinacion de letras, identica al file buscado
                 Return:
                     Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
                     caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
                     para el filename
                 """
                 fullfilename = None
                 find_flag = False
                 filename = None
                 prefixDirList = [None,'d','D']
                 if ext.lower() == ".r": #voltage
                     prefixFileList = ['d','D']
                 elif ext.lower() == ".pdata": #spectra
                     prefixFileList = ['p','P']
                 else:
                     return None, filename
                 #barrido por las combinaciones posibles
                 for prefixDir in prefixDirList:
                     thispath = path
                     if prefixDir != None:
                         #formo el nombre del directorio xYYYYDDD (x=d o x=D)
                         if foldercounter == 0:
                             thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
                         else:
                             thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
                     for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
                         filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
                         fullfilename = os.path.join( thispath, filename ) #formo el path completo
                         if os.path.exists( fullfilename ): #verifico que exista
                             find_flag = True
                             break
                     if find_flag:
                         break
                 if not(find_flag):
                     return None, filename
                 return fullfilename, filename
             def isRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return 0
                 return 1
             def isRadarFile(file):
                     try:
                         year = int(file[1:5])
                         doy = int(file[5:8])
                         set = int(file[8:11])
                     except:
                         return 0
                     return 1
             def getDateFromRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
                 return thisDate
             def getDateFromRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
                 return thisDate
             class JRODataIO:
                 c = 3E8
                 isConfig = False
                 basicHeaderObj = None
                 systemHeaderObj = None
                 radarControllerHeaderObj = None
                 processingHeaderObj = None
                 dtype = None
                 pathList = []
                 filenameList = []
                 filename = None
                 ext = None
                 flagIsNewFile = 1
                 flagDiscontinuousBlock = 0
                 flagIsNewBlock = 0
                 fp = None
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 versionFile = 1103
                 fileSize = None
             #     ippSeconds = None
                 fileSizeByHeader = None
                 fileIndex = None
                 profileIndex = None
                 blockIndex = None
                 nTotalBlocks = None
                 maxTimeStep = 30
                 lastUTTime = None
                 datablock = None
                 dataOut = None
                 blocksize = None
                 getByBlock = False
                 def __init__(self):
                     raise NotImplementedError
                 def run(self):
                     raise NotImplementedError
                 def getDtypeWidth(self):
                     dtype_index = get_dtype_index(self.dtype)
                     dtype_width = get_dtype_width(dtype_index)
                     return dtype_width
             class JRODataReader(JRODataIO):
                 online = 0
                 realtime = 0
                 nReadBlocks = 0
                 delay  = 10   #number of seconds waiting a new file
                 nTries  = 3  #quantity tries
                 nFiles = 3   #number of files for searching
                 path = None
                 foldercounter = 0
                 flagNoMoreFiles = 0
                 datetimeList = []
                 __isFirstTimeOnline = 1
                 __printInfo = True
                 profileIndex = None
                 nTxs = 1
                 txIndex = None
                 #Added--------------------
                 selBlocksize = None
                 selBlocktime = None
                 def __init__(self):
                     """
                     This class is used to find data files
                     Example:
                         reader = JRODataReader()
                         fileList = reader.findDataFiles()
                     """
                     pass
                 def createObjByDefault(self):
                     """
                     """
                     raise NotImplementedError
                 def getBlockDimension(self):
                     raise NotImplementedError
                 def __searchFilesOffLine(self,
                                         path,
                                         startDate=None,
                                         endDate=None,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.r',
+                                        queue=None,
+                                        cursor=None,
+                                        skip=None,
                                         walk=True):
                     self.filenameList = []
                     self.datetimeList = []
                     pathList = []
                     dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
                     if dateList == []:
             #             print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
                         return None, None
                     if len(dateList) > 1:
                         print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
                     else:
                         print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
                     filenameList = []
                     datetimeList = []
                     for thisPath in pathList:
             #             thisPath = pathList[pathDict[file]]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
-                        for file in fileList:
+                        skippedFileList = []
+                        if cursor is not None and skip is not None:
+                            # if cursor*skip > len(fileList):
+                            if skip == 0:
+                                if queue is not None:
+                                    queue.put(len(fileList))
+                                skippedFileList = []
+                            else:
+                                skippedFileList = fileList[cursor*skip: cursor*skip + skip]
+                        else:
+                            skippedFileList = fileList
+                        for file in skippedFileList:
                             filename = os.path.join(thisPath,file)
                             if not isFileInDateRange(filename, startDate, endDate):
                                 continue
                             thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
                         return None, None
                     print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
                     """
                     Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
                     devuelve el archivo encontrado ademas de otros datos.
                     Input:
                         path           :    carpeta donde estan contenidos los files que contiene data
                         expLabel        :     Nombre del subexperimento (subfolder)
                         ext              :    extension de los files
                         walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                     Return:
                         directory   :    eL directorio donde esta el file encontrado
                         filename    :    el ultimo file de una determinada carpeta
                         year        :    el anho
                         doy         :    el numero de dia del anho
                         set         :    el set del archivo
                     """
                     if not os.path.isdir(path):
                         return None, None, None, None, None, None
                     dirList = []
                     if not walk:
                         fullpath = path
                         foldercounter = 0
                     else:
                         #Filtra solo los directorios
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isRadarFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None, None, None, None, None
                         dirList = sorted( dirList, key=str.lower )
                         doypath = dirList[-1]
                         foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
                         fullpath = os.path.join(path, doypath, expLabel)
                     print "[Reading] %s folder was found: " %(fullpath )
                     if set == None:
                         filename = getlastFileFromPath(fullpath, ext)
                     else:
                         filename = getFileFromSet(fullpath, ext, set)
                     if not(filename):
                         return None, None, None, None, None, None
                     print "[Reading] %s file was found" %(filename)
                     if not(self.__verifyFile(os.path.join(fullpath, filename))):
                         return None, None, None, None, None, None
                     year = int( filename[1:5] )
                     doy  = int( filename[5:8] )
                     set  = int( filename[8:11] )
                     return fullpath, foldercounter, filename, year, doy, set
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
             #                 print "[Reading] No more Files"
                             return 0
                         filename = self.filenameList[idFile]
                         if not(self.__verifyFile(filename)):
                             continue
                         fileSize = os.path.getsize(filename)
                         fp = open(filename,'rb')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fp = fp
             #         print "[Reading] Setting the file: %s"%self.filename
                     return 1
                 def __setNextFileOnline(self):
                     """
                     Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
                     no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
                     siguientes.
                     Affected:
                         self.flagIsNewFile
                         self.filename
                         self.fileSize
                         self.fp
                         self.set
                         self.flagNoMoreFiles
                     Return:
 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
 : si el file fue abierto con exito y esta listo a ser leido
                     Excepciones:
                         Si un determinado file no puede ser abierto
                     """
                     nFiles = 0
                     fileOk_flag = False
                     firstTime_flag = True
                     self.set += 1
                     if self.set > 999:
                         self.set = 0
                         self.foldercounter += 1
                     #busca el 1er file disponible
                     fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                     if fullfilename:
                         if self.__verifyFile(fullfilename, False):
                             fileOk_flag = True
                     #si no encuentra un file entonces espera y vuelve a buscar
                     if not(fileOk_flag):
                         for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
                             if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
                                 tries = self.nTries
                             else:
                                 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
                             for nTries in range( tries ):
                                 if firstTime_flag:
                                     print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
                                     sleep( self.delay )
                                 else:
                                     print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
                                 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                                 if fullfilename:
                                     if self.__verifyFile(fullfilename):
                                         fileOk_flag = True
                                         break
                             if fileOk_flag:
                                 break
                             firstTime_flag = False
                             print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
                             self.set += 1
                             if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
                                 self.set = 0
                                 self.doy += 1
                                 self.foldercounter = 0
                     if fileOk_flag:
                         self.fileSize = os.path.getsize( fullfilename )
                         self.filename = fullfilename
                         self.flagIsNewFile = 1
                         if self.fp != None: self.fp.close()
                         self.fp = open(fullfilename, 'rb')
                         self.flagNoMoreFiles = 0
             #             print '[Reading] Setting the file: %s' % fullfilename
                     else:
                         self.fileSize = 0
                         self.filename = None
                         self.flagIsNewFile = 0
                         self.fp = None
                         self.flagNoMoreFiles = 1
             #             print '[Reading] No more files to read'
                     return fileOk_flag
                 def setNextFile(self):
                     if self.fp != None:
                         self.fp.close()
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         print '[Reading] No more files to read'
                         return 0
                     print '[Reading] Setting the file: %s' % self.filename
                     self.__readFirstHeader()
                     self.nReadBlocks = 0
                     return 1
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.basicHeaderObj.read(self.fp)
                             return 1
                         if self.fileSize == self.fileSizeByHeader:
             #                self.flagEoF = True
                             return 0
                         print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         sleep( self.delay )
                     return 0
                 def waitDataBlock(self,pointer_location):
                     currentPointer = pointer_location
                     neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             return 1
                         print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         sleep( self.delay )
                     return 0
                 def __jumpToLastBlock(self):
                     if not(self.__isFirstTimeOnline):
                         return
                     csize = self.fileSize - self.fp.tell()
                     blocksize = self.processingHeaderObj.blockSize
                     #salta el primer bloque de datos
                     if csize > self.processingHeaderObj.blockSize:
                         self.fp.seek(self.fp.tell() + blocksize)
                     else:
                         return
                     csize = self.fileSize - self.fp.tell()
                     neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     while True:
                         if self.fp.tell()<self.fileSize:
                             self.fp.seek(self.fp.tell() + neededsize)
                         else:
                             self.fp.seek(self.fp.tell() - neededsize)
                             break
             #        csize = self.fileSize - self.fp.tell()
             #        neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
             #        factor = int(csize/neededsize)
             #        if factor > 0:
             #            self.fp.seek(self.fp.tell() + factor*neededsize)
                     self.flagIsNewFile = 0
                     self.__isFirstTimeOnline = 0
                 def __setNewBlock(self):
                     if self.fp == None:
                         return 0
             #         if self.online:
             #             self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if self.realtime:
                         self.flagDiscontinuousBlock = 1
                         if not(self.setNextFile()):
                             return 0
                         else:
                             return 1
                     currentSize = self.fileSize - self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     if (currentSize >= neededSize):
                         self.basicHeaderObj.read(self.fp)
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if self.__waitNewBlock():
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
                     self.lastUTTime = self.basicHeaderObj.utc
                     self.flagDiscontinuousBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagDiscontinuousBlock = 1
                     return 1
                 def readNextBlock(self):
                     #Skip block out of startTime and endTime
                     while True:
                         if not(self.__setNewBlock()):
                             return 0
                         if not(self.readBlock()):
                             return 0
                         self.getBasicHeader()
                         if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
                             print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
                                                                           self.processingHeaderObj.dataBlocksPerFile,
                                                                           self.dataOut.datatime.ctime())
                             continue
                         break
                     print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
                                                               self.processingHeaderObj.dataBlocksPerFile,
                                                               self.dataOut.datatime.ctime())
                     return 1
                 def __readFirstHeader(self):
                     self.basicHeaderObj.read(self.fp)
                     self.systemHeaderObj.read(self.fp)
                     self.radarControllerHeaderObj.read(self.fp)
                     self.processingHeaderObj.read(self.fp)
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
                     else:
                         raise ValueError, 'Data type was not defined'
                     self.dtype = datatype_str
                     #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
             #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
             #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def __verifyFile(self, filename, msgFlag=True):
                     msg = None
                     try:
                         fp = open(filename, 'rb')
                     except IOError:
                         if msgFlag:
                             print "[Reading] File %s can't be opened" % (filename)
                         return False
                     currentPosition = fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                     if neededSize == 0:
                         basicHeaderObj = BasicHeader(LOCALTIME)
                         systemHeaderObj = SystemHeader()
                         radarControllerHeaderObj = RadarControllerHeader()
                         processingHeaderObj = ProcessingHeader()
                         if not( basicHeaderObj.read(fp) ):
                             fp.close()
                             return False
                         if not( systemHeaderObj.read(fp) ):
                             fp.close()
                             return False
                         if not( radarControllerHeaderObj.read(fp) ):
                             fp.close()
                             return False
                         if not( processingHeaderObj.read(fp) ):
                             fp.close()
                             return False
                         neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
                     else:
                         msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
                     fp.close()
                     fileSize = os.path.getsize(filename)
                     currentSize = fileSize - currentPosition
                     if currentSize < neededSize:
                         if msgFlag and (msg != None):
                             print msg
                         return False
                     return True
                 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
                     path_empty = True
                     dateList = []
                     pathList = []
                     multi_path = path.split(',')
                     if not walk:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             fileList = glob.glob1(single_path, "*"+ext)
                             if not fileList:
                                 continue
                             path_empty = False
                             fileList.sort()
                             for thisFile in fileList:
                                 if not os.path.isfile(os.path.join(single_path, thisFile)):
                                     continue
                                 if not isRadarFile(thisFile):
                                     continue
                                 if not isFileInDateRange(thisFile, startDate, endDate):
                                     continue
                                 thisDate = getDateFromRadarFile(thisFile)
                                 if thisDate in dateList:
                                     continue
                                 dateList.append(thisDate)
                                 pathList.append(single_path)
                     else:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             dirList = []
                             for thisPath in os.listdir(single_path):
                                 if not os.path.isdir(os.path.join(single_path,thisPath)):
                                     continue
                                 if not isRadarFolder(thisPath):
                                     continue
                                 if not isFolderInDateRange(thisPath, startDate, endDate):
                                     continue
                                 dirList.append(thisPath)
                             if not dirList:
                                 continue
                             dirList.sort()
                             for thisDir in dirList:
                                 datapath = os.path.join(single_path, thisDir, expLabel)
                                 fileList = glob.glob1(datapath, "*"+ext)
                                 if not fileList:
                                     continue
                                 path_empty = False
                                 thisDate = getDateFromRadarFolder(thisDir)
                                 pathList.append(datapath)
                                 dateList.append(thisDate)
                     dateList.sort()
                     if walk:
                         pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
                     else:
                         pattern_path = multi_path[0]
                     if path_empty:
                         print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
                     else:
                         if not dateList:
                             print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
                     if include_path:
                         return dateList, pathList
                     return dateList
                 def setup(self,
                             path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=None,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True,
                             getblock = False,
                             nTxs = 1,
                             realtime=False,
                             blocksize=None,
-                            blocktime=None):
+                            blocktime=None,
+                            queue=None,
+                            skip=None,
+                            cursor=None):
                     if path == None:
                         raise ValueError, "[Reading] The path is not valid"
                     if ext == None:
                         ext = self.ext
                     if online:
                         print "[Reading] Searching files in online mode..."
                         for nTries in range( self.nTries ):
                             fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
                             if fullpath:
                                 break
                             print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
                             sleep( self.delay )
                         if not(fullpath):
                             print "[Reading] There 'isn't any valid file in %s" % path
                             return
                         self.year = year
                         self.doy  = doy
                         self.set  = set - 1
                         self.path = path
                         self.foldercounter = foldercounter
                         last_set = None
                     else:
                         print "[Reading] Searching files in offline mode ..."
                         pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
-                                                                           walk=walk)
+                                                                           walk=walk, cursor=cursor,
+                                                                           skip=skip, queue=queue)
                         if not(pathList):
             #                 print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
             #                                                         datetime.datetime.combine(startDate,startTime).ctime(),
             #                                                         datetime.datetime.combine(endDate,endTime).ctime())
             #                 sys.exit(-1)
                             self.fileIndex = -1
                             self.pathList = []
                             self.filenameList = []
                             return
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                         file_name = os.path.basename(filenameList[-1])
                         basename, ext = os.path.splitext(file_name)
                         last_set = int(basename[-3:])
                     self.online = online
                     self.realtime = realtime
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     self.getByBlock = getblock
                     self.nTxs = nTxs
                     self.startTime = startTime
                     self.endTime = endTime
                     #Added-----------------
                     self.selBlocksize = blocksize
                     self.selBlocktime = blocktime
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         elif startDate != None:
                             print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                         else:
                             print "[Reading] No files"
                             self.fileIndex = -1
                             self.pathList = []
                             self.filenameList = []
                             return
             #         self.getBasicHeader()
                     if last_set != None:
                         self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
                     return
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                     self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
             #         self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
                 def getFirstHeader(self):
                     raise NotImplementedError
                 def getData(self):
                     raise NotImplementedError
                 def hasNotDataInBuffer(self):
                     raise NotImplementedError
                 def readBlock(self):
                     raise NotImplementedError
                 def isEndProcess(self):
                     return self.flagNoMoreFiles
                 def printReadBlocks(self):
                     print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
                 def printTotalBlocks(self):
                     print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
                 def printNumberOfBlock(self):
                     if self.flagIsNewBlock:
                         print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
                                                                   self.processingHeaderObj.dataBlocksPerFile,
                                                                   self.dataOut.datatime.ctime())
                 def printInfo(self):
                     if self.__printInfo == False:
                         return
                     self.basicHeaderObj.printInfo()
                     self.systemHeaderObj.printInfo()
                     self.radarControllerHeaderObj.printInfo()
                     self.processingHeaderObj.printInfo()
                     self.__printInfo = False
                 def run(self, **kwargs):
                     if not(self.isConfig):
             #            self.dataOut = dataOut
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             class JRODataWriter(JRODataIO):
                 """
                 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 blockIndex = 0
                 path = None
                 setFile = None
                 profilesPerBlock = None
                 blocksPerFile = None
                 nWriteBlocks = 0
                 fileDate = None
                 def __init__(self, dataOut=None):
                     raise NotImplementedError
                 def hasAllDataInBuffer(self):
                     raise NotImplementedError
                 def setBlockDimension(self):
                     raise NotImplementedError
                 def writeBlock(self):
                     raise NotImplementedError
                 def putData(self):
                     raise NotImplementedError
                 def getProcessFlags(self):
                     processFlags = 0
                     dtype_index = get_dtype_index(self.dtype)
                     procflag_dtype = get_procflag_dtype(dtype_index)
                     processFlags += procflag_dtype
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code is not None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nCohInt > 1:
                         processFlags += PROCFLAG.COHERENT_INTEGRATION
                     if self.dataOut.type == "Spectra":
                         if self.dataOut.nIncohInt > 1:
                             processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                         if self.dataOut.data_dc is not None:
                             processFlags += PROCFLAG.SAVE_CHANNELS_DC
                         if self.dataOut.flagShiftFFT:
                             processFlags += PROCFLAG.SHIFT_FFT_DATA
                     return processFlags
                 def setBasicHeader(self):
                     self.basicHeaderObj.size = self.basicHeaderSize #bytes
                     self.basicHeaderObj.version = self.versionFile
                     self.basicHeaderObj.dataBlock = self.nTotalBlocks
                     utc = numpy.floor(self.dataOut.utctime)
                     milisecond  = (self.dataOut.utctime - utc)* 1000.0
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = milisecond
                     self.basicHeaderObj.timeZone = self.dataOut.timeZone
                     self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                     self.basicHeaderObj.errorCount = self.dataOut.errorCount
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj self.
                     Return:
                         None
                     """
                     raise NotImplementedError
                 def __writeFirstHeader(self):
                     """
                     Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                     Affected:
                         __dataType
                     Return:
                         None
                     """
             #        CALCULAR PARAMETROS
                     sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                     self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                     self.basicHeaderObj.write(self.fp)
                     self.systemHeaderObj.write(self.fp)
                     self.radarControllerHeaderObj.write(self.fp)
                     self.processingHeaderObj.write(self.fp)
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                     """
                     if self.fp == None:
                         self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                         self.basicHeaderObj.write(self.fp)
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     """
                     Selecciona el bloque siguiente de datos y los escribe en un file
                     Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                     """
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     print "[Writing] Block No. %d/%d" %(self.blockIndex,
                                                         self.processingHeaderObj.dataBlocksPerFile)
                     return 1
                 def setNextFile(self):
                     """
                     Determina el siguiente file que sera escrito
                     Affected:
                         self.filename
                         self.subfolder
                         self.fp
                         self.setFile
                         self.flagIsNewFile
                     Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                     """
                     ext = self.ext
                     path = self.path
                     if self.fp != None:
                         self.fp.close()
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     setFile = self.setFile
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     setFile += 1
                     #If this is a new day it resets some values
                     if self.dataOut.datatime.date() > self.fileDate:
                         setFile = 0
                         self.nTotalBlocks = 0
                     filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
                     filename = os.path.join( path, subfolder, filen )
                     fp = open( filename,'wb' )
                     self.blockIndex = 0
                     #guardando atributos
                     self.filename = filename
                     self.subfolder = subfolder
                     self.fp = fp
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     self.fileDate = self.dataOut.datatime.date()
                     self.setFirstHeader()
                     print '[Writing] Opening file: %s'%self.filename
                     self.__writeFirstHeader()
                     return 1
                 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
                     """
                     Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                     Inputs:
                         path                :    directory where data will be saved
                         profilesPerBlock    :    number of profiles per block
                         set                 :    initial file set
                         datatype            :    An integer number that defines data type:
 : int8  (1 byte)
 : int16 (2 bytes)
 : int32 (4 bytes)
 : int64 (8 bytes)
 : float32 (4 bytes)
 : double64 (8 bytes)
                     Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                     """
                     if ext == None:
                         ext = self.ext
                     self.ext = ext.lower()
                     self.path = path
                     if set is None:
                         self.setFile = -1
                     else:
                         self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile
                     self.profilesPerBlock = profilesPerBlock
                     self.dataOut = dataOut
                     self.fileDate = self.dataOut.datatime.date()
                     #By default
                     self.dtype = self.dataOut.dtype
                     if datatype is not None:
                         self.dtype = get_numpy_dtype(datatype)
                     if not(self.setNextFile()):
                         print "[Writing] There isn't a next file"
                         return 0
                     self.setBlockDimension()
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
-                    self.putData()
+                    self.putData()

schainpy/model/utils/jroutils_publish.py +3 -3

             '''
             @author: Juan C. Espinoza
             '''
             import time
             import json
             import numpy
             import paho.mqtt.client as mqtt
             import zmq
             import cPickle as pickle
             import datetime
             from zmq.utils.monitor import recv_monitor_message
             from functools import wraps
             from threading import Thread
             from multiprocessing import Process
             from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
             MAXNUMX = 100
             MAXNUMY = 100
             throttle_value = 5
             class PrettyFloat(float):
                 def __repr__(self):
                     return '%.2f' % self
             def roundFloats(obj):
                 if isinstance(obj, list):
                     return map(roundFloats, obj)
                 elif isinstance(obj, float):
                     return round(obj, 2)
             class throttle(object):
                 """Decorator that prevents a function from being called more than once every
                 time period.
                 To create a function that cannot be called more than once a minute, but
                 will sleep until it can be called:
                 @throttle(minutes=1)
                 def foo():
                   pass
                 for i in range(10):
                   foo()
                   print "This function has run %s times." % i
                 """
                 def __init__(self, seconds=0, minutes=0, hours=0):
                     self.throttle_period = datetime.timedelta(
                         seconds=seconds, minutes=minutes, hours=hours
                     )
                     self.time_of_last_call = datetime.datetime.min
                 def __call__(self, fn):
                     @wraps(fn)
                     def wrapper(*args, **kwargs):
                         now = datetime.datetime.now()
                         time_since_last_call = now - self.time_of_last_call
                         time_left = self.throttle_period - time_since_last_call
                         if time_left > datetime.timedelta(seconds=0):
                             return
                         self.time_of_last_call = datetime.datetime.now()
                         return fn(*args, **kwargs)
                     return wrapper
             class PublishData(Operation):
                 """Clase publish."""
                 def __init__(self, **kwargs):
                     """Inicio."""
                     Operation.__init__(self, **kwargs)
                     self.isConfig = False
                     self.client = None
                     self.zeromq = None
                     self.mqtt = None
                 def on_disconnect(self, client, userdata, rc):
                     if rc != 0:
                         print("Unexpected disconnection.")
                     self.connect()
                 def connect(self):
                     print 'trying to connect'
                     try:
                         self.client.connect(
                             host=self.host,
                             port=self.port,
                             keepalive=60*10,
                             bind_address='')
                         print "connected"
                         self.client.loop_start()
                         # self.client.publish(
                         #     self.topic + 'SETUP',
                         #     json.dumps(setup),
                         #     retain=True
                         #     )
                     except:
                         print "MQTT Conection error."
                         self.client = False
                 def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, **kwargs):
                     self.counter = 0
                     self.topic = kwargs.get('topic', 'schain')
                     self.delay = kwargs.get('delay', 0)
                     self.plottype = kwargs.get('plottype', 'spectra')
                     self.host = kwargs.get('host', "10.10.10.82")
                     self.port = kwargs.get('port', 3000)
                     self.clientId = clientId
                     self.cnt = 0
                     self.zeromq = zeromq
                     self.mqtt = kwargs.get('plottype', 0)
                     self.client = None
                     setup = []
                     if mqtt is 1:
                         print 'mqqt es 1'
                         self.client = mqtt.Client(
                             client_id=self.clientId + self.topic + 'SCHAIN',
                             clean_session=True)
                         self.client.on_disconnect = self.on_disconnect
                         self.connect()
                         for plot in self.plottype:
                             setup.append({
                                 'plot': plot,
                                 'topic': self.topic + plot,
                                 'title': getattr(self, plot + '_' + 'title', False),
                                 'xlabel': getattr(self, plot + '_' + 'xlabel', False),
                                 'ylabel': getattr(self, plot + '_' + 'ylabel', False),
                                 'xrange': getattr(self, plot + '_' + 'xrange', False),
                                 'yrange': getattr(self, plot + '_' + 'yrange', False),
                                 'zrange': getattr(self, plot + '_' + 'zrange', False),
                             })
                     if zeromq is 1:
                         context = zmq.Context()
                         self.zmq_socket = context.socket(zmq.PUSH)
                         server = kwargs.get('server', 'zmq.pipe')
                         if 'tcp://' in server:
                             address = server
                         else:
                             address = 'ipc:///tmp/%s' % server
                         self.zmq_socket.connect(address)
                         time.sleep(1)
                         print 'zeromq configured'
                 def publish_data(self):
                     self.dataOut.finished = False
                     if self.mqtt is 1:
                         yData = self.dataOut.heightList[:2].tolist()
                         if self.plottype == 'spectra':
                             data = getattr(self.dataOut, 'data_spc')
                             z = data/self.dataOut.normFactor
                             zdB = 10*numpy.log10(z)
                             xlen, ylen = zdB[0].shape
                             dx = int(xlen/MAXNUMX) + 1
                             dy = int(ylen/MAXNUMY) + 1
                             Z = [0 for i in self.dataOut.channelList]
                             for i in self.dataOut.channelList:
                                 Z[i] = zdB[i][::dx, ::dy].tolist()
                             payload = {
                                 'timestamp': self.dataOut.utctime,
                                 'data': roundFloats(Z),
                                 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                 'interval': self.dataOut.getTimeInterval(),
                                 'type': self.plottype,
                                 'yData': yData
                             }
                             # print payload
                         elif self.plottype in ('rti', 'power'):
                             data = getattr(self.dataOut, 'data_spc')
                             z = data/self.dataOut.normFactor
                             avg = numpy.average(z, axis=1)
                             avgdB = 10*numpy.log10(avg)
                             xlen, ylen = z[0].shape
                             dy = numpy.floor(ylen/self.__MAXNUMY) + 1
                             AVG = [0 for i in self.dataOut.channelList]
                             for i in self.dataOut.channelList:
                                 AVG[i] = avgdB[i][::dy].tolist()
                             payload = {
                                 'timestamp': self.dataOut.utctime,
                                 'data': roundFloats(AVG),
                                 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                 'interval': self.dataOut.getTimeInterval(),
                                 'type': self.plottype,
                                 'yData': yData
                             }
                         elif self.plottype == 'noise':
                             noise = self.dataOut.getNoise()/self.dataOut.normFactor
                             noisedB = 10*numpy.log10(noise)
                             payload = {
                                 'timestamp': self.dataOut.utctime,
                                 'data': roundFloats(noisedB.reshape(-1, 1).tolist()),
                                 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                 'interval': self.dataOut.getTimeInterval(),
                                 'type': self.plottype,
                                 'yData': yData
                             }
                         elif self.plottype == 'snr':
                             data = getattr(self.dataOut, 'data_SNR')
                             avgdB = 10*numpy.log10(data)
                             ylen = data[0].size
                             dy = numpy.floor(ylen/self.__MAXNUMY) + 1
                             AVG = [0 for i in self.dataOut.channelList]
                             for i in self.dataOut.channelList:
                                 AVG[i] = avgdB[i][::dy].tolist()
                             payload = {
                                 'timestamp': self.dataOut.utctime,
                                 'data': roundFloats(AVG),
                                 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                 'type': self.plottype,
                                 'yData': yData
                             }
                         else:
                             print "Tipo de grafico invalido"
                             payload = {
                                 'data': 'None',
                                 'timestamp': 'None',
                                 'type': None
                             }
                             # print 'Publishing data to {}'.format(self.host)
                         self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
                     if self.zeromq is 1:
                         print '[Sending] {} - {}'.format(self.dataOut.type, self.dataOut.datatime)
                         self.zmq_socket.send_pyobj(self.dataOut)
                 def run(self, dataOut, **kwargs):
                     self.dataOut = dataOut
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.publish_data()
                     time.sleep(self.delay)
                 def close(self):
                     if self.zeromq is 1:
                         self.dataOut.finished = True
                         self.zmq_socket.send_pyobj(self.dataOut)
                     if self.client:
                         self.client.loop_stop()
                         self.client.disconnect()
             class ReceiverData(ProcessingUnit, Process):
                 def __init__(self, **kwargs):
                     ProcessingUnit.__init__(self, **kwargs)
                     Process.__init__(self)
                     self.mp = False
                     self.isConfig = False
                     self.plottypes =[]
                     self.connections = 0
                     server = kwargs.get('server', 'zmq.pipe')
                     if 'tcp://' in server:
                         address = server
                     else:
                         address = 'ipc:///tmp/%s' % server
                     self.address = address
                     self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
                     self.realtime = kwargs.get('realtime', False)
                     global throttle_value
                     throttle_value = kwargs.get('throttle', 10)
                     self.setup()
                 def setup(self):
                     self.data = {}
                     self.data['times'] = []
                     for plottype in self.plottypes:
                         self.data[plottype] = {}
+                    self.data['noise'] = {}
                     self.isConfig = True
                 def event_monitor(self, monitor):
                     events = {}
                     for name in dir(zmq):
                         if name.startswith('EVENT_'):
                             value = getattr(zmq, name)
                             events[value] = name
                     while monitor.poll():
                         evt = recv_monitor_message(monitor)
                         if evt['event'] == 32:
                             self.connections += 1
                         if evt['event'] == 512:
                             pass
                         if self.connections == 0 and self.started is True:
                             self.ended = True
                             # send('ENDED')
                         evt.update({'description': events[evt['event']]})
                         if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
                             break
                     monitor.close()
                     print("event monitor thread done!")
                 @throttle(seconds=throttle_value)
                 def sendData(self, data):
                     self.send(data)
                 def send(self, data):
                     print '[sending] data=%s size=%s' % (data.keys(), len(data['times']))
                     self.sender.send_pyobj(data)
                 def update(self):
                     t = self.dataOut.ltctime
                     self.data['times'].append(t)
                     self.data['dataOut'] = self.dataOut
                     for plottype in self.plottypes:
                         if plottype == 'spc':
                             z = self.dataOut.data_spc/self.dataOut.normFactor
-                            zdB = 10*numpy.log10(z)
+                            self.data[plottype] = 10*numpy.log10(z)
-                            self.data[plottype] = zdB
+                            self.data['noise'][t] = 10*numpy.log10(self.dataOut.getNoise()/self.dataOut.normFactor)
                         if plottype == 'rti':
                             self.data[plottype][t] = self.dataOut.getPower()
                         if plottype == 'snr':
                             self.data[plottype][t] = 10*numpy.log10(self.dataOut.data_SNR)
                         if plottype == 'dop':
                             self.data[plottype][t] = 10*numpy.log10(self.dataOut.data_DOP)
                         if plottype == 'coh':
                             self.data[plottype][t] = self.dataOut.getCoherence()
                         if plottype == 'phase':
                             self.data[plottype][t] = self.dataOut.getCoherence(phase=True)
                 def run(self):
                     print '[Starting] {} from {}'.format(self.name, self.address)
                     self.context = zmq.Context()
                     self.receiver = self.context.socket(zmq.PULL)
                     self.receiver.bind(self.address)
                     monitor = self.receiver.get_monitor_socket()
                     self.sender = self.context.socket(zmq.PUB)
                     self.sender.bind("ipc:///tmp/zmq.plots")
                     t = Thread(target=self.event_monitor, args=(monitor,))
                     t.start()
                     while True:
                         self.dataOut = self.receiver.recv_pyobj()
                         print '[Receiving] {} - {}'.format(self.dataOut.type,
                                                            self.dataOut.datatime.ctime())
                         self.update()
                         if self.dataOut.finished is True:
                             self.send(self.data)
                             self.connections -= 1
                             if self.connections==0 and self.started:
                                 self.ended = True
                                 self.data['ENDED'] = True
                                 self.send(self.data)
                                 self.setup()
                         else:
                             if self.realtime:
                                 self.send(self.data)
                             else:
                                 self.sendData(self.data)
                             self.started = True
                     return

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages