schain-jc Commit - r931:9d13aae43f1b

finishing day, need testing

José Chávez -

r931:9d13aae43f1b

parent child

Context file:

r931:9d13aae43f1b

Collapse all files

schainpy/controller.py +2 -2

             '''
             Created on September , 2012
             @author:
             '''
             import sys
             import ast
             import datetime
             import traceback
             import math
+            import time
             from multiprocessing import Process, Queue, cpu_count
             import schainpy
             import schainpy.admin
             from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
             from xml.dom import minidom
             from schainpy.model import *
             from time import sleep
             def prettify(elem):
                 """Return a pretty-printed XML string for the Element.
                 """
                 rough_string = tostring(elem, 'utf-8')
                 reparsed = minidom.parseString(rough_string)
                 return reparsed.toprettyxml(indent="  ")
             def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
                 skip = 0
                 cursor = 0
                 nFiles = None
                 processes = []
                 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
                 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
                 days = (dt2 - dt1).days
                 for day in range(days+1):
                     skip = 0
                     cursor = 0
                     q = Queue()
                     processes = []
                     dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
                     firstProcess = Process(target=child, args=(cursor, skip, q, dt))
                     firstProcess.start()
                     if by_day:
                         continue
                     nFiles = q.get()
                     firstProcess.terminate()
                     skip = int(math.ceil(nFiles/nProcess))
                     while True:
                         processes.append(Process(target=child, args=(cursor, skip, q, dt)))
                         processes[cursor].start()
                         if nFiles < cursor*skip:
                             break
                         cursor += 1
                     def beforeExit(exctype, value, trace):
                         for process in processes:
                             process.terminate()
                             process.join()
                         print traceback.print_tb(trace)
                     sys.excepthook = beforeExit
                     for process in processes:
                         process.join()
                         process.terminate()
-                    sys.exit()
+                    time.sleep(3)
             class ParameterConf():
                 id = None
                 name = None
                 value = None
                 format = None
                 __formated_value = None
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.format = 'str'
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getValue(self):
                     value = self.value
                     format = self.format
                     if self.__formated_value != None:
                         return self.__formated_value
                     if format == 'obj':
                         return value
                     if format == 'str':
                         self.__formated_value = str(value)
                         return self.__formated_value
                     if value == '':
                         raise ValueError, "%s: This parameter value is empty" %self.name
                     if format == 'list':
                         strList = value.split(',')
                         self.__formated_value = strList
                         return self.__formated_value
                     if format == 'intlist':
                         """
                         Example:
                             value = (0,1,2)
                         """
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             new_value = [int(new_value)]
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'floatlist':
                         """
                         Example:
                             value = (0.5, 1.4, 2.7)
                         """
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             new_value = [float(new_value)]
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'date':
                         strList = value.split('/')
                         intList = [int(x) for x in strList]
                         date = datetime.date(intList[0], intList[1], intList[2])
                         self.__formated_value = date
                         return self.__formated_value
                     if format == 'time':
                         strList = value.split(':')
                         intList = [int(x) for x in strList]
                         time = datetime.time(intList[0], intList[1], intList[2])
                         self.__formated_value = time
                         return self.__formated_value
                     if format == 'pairslist':
                         """
                         Example:
                             value = (0,1),(1,2)
                         """
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             raise ValueError, "%s has to be a tuple or list of pairs" %value
                         if type(new_value[0]) not in (tuple, list):
                             if len(new_value) != 2:
                                 raise ValueError, "%s has to be a tuple or list of pairs" %value
                             new_value = [new_value]
                         for thisPair in new_value:
                             if len(thisPair) != 2:
                                 raise ValueError, "%s has to be a tuple or list of pairs" %value
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'multilist':
                         """
                         Example:
                             value = (0,1,2),(3,4,5)
                         """
                         multiList = ast.literal_eval(value)
                         if type(multiList[0]) == int:
                             multiList = ast.literal_eval("(" + value + ")")
                         self.__formated_value = multiList
                         return self.__formated_value
                     if format == 'bool':
                         value = int(value)
                     if format == 'int':
                         value = float(value)
                     format_func = eval(format)
                     self.__formated_value = format_func(value)
                     return self.__formated_value
                 def updateId(self, new_id):
                     self.id = str(new_id)
                 def setup(self, id, name, value, format='str'):
                     self.id = str(id)
                     self.name = name
                     if format == 'obj':
                         self.value = value
                     else:
                         self.value = str(value)
                     self.format = str.lower(format)
                     self.getValue()
                     return 1
                 def update(self, name, value, format='str'):
                     self.name = name
                     self.value = str(value)
                     self.format = format
                 def makeXml(self, opElement):
                     if self.name not in ('queue',):
                         parmElement = SubElement(opElement, self.ELEMENTNAME)
                         parmElement.set('id', str(self.id))
                         parmElement.set('name', self.name)
                         parmElement.set('value', self.value)
                         parmElement.set('format', self.format)
                 def readXml(self, parmElement):
                     self.id = parmElement.get('id')
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                     self.format = str.lower(parmElement.get('format'))
                     #Compatible with old signal chain version
                     if self.format == 'int' and self.name == 'idfigure':
                         self.name = 'id'
                 def printattr(self):
                     print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
             class OperationConf():
                 id = None
                 name = None
                 priority = None
                 type = None
                 parmConfObjList = []
                 ELEMENTNAME = 'Operation'
                 def __init__(self):
                     self.id = '0'
                     self.name = None
                     self.priority = None
                     self.type = 'self'
                 def __getNewId(self):
                     return int(self.id)*10 + len(self.parmConfObjList) + 1
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     n = 1
                     for parmObj in self.parmConfObjList:
                         idParm = str(int(new_id)*10 + n)
                         parmObj.updateId(idParm)
                         n += 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getParameterObjList(self):
                     return self.parmConfObjList
                 def getParameterObj(self, parameterName):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.name != parameterName:
                             continue
                         return parmConfObj
                     return None
                 def getParameterObjfromValue(self, parameterValue):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.getValue() != parameterValue:
                             continue
                         return parmConfObj.getValue()
                     return None
                 def getParameterValue(self, parameterName):
                     parameterObj = self.getParameterObj(parameterName)
             #         if not parameterObj:
             #             return None
                     value = parameterObj.getValue()
                     return value
                 def getKwargs(self):
                     kwargs = {}
                     for parmConfObj in self.parmConfObjList:
                         if self.name == 'run' and parmConfObj.name == 'datatype':
                             continue
                         kwargs[parmConfObj.name] = parmConfObj.getValue()
                     return kwargs
                 def setup(self, id, name, priority, type):
                     self.id = str(id)
                     self.name = name
                     self.type = type
                     self.priority = priority
                     self.parmConfObjList = []
                 def removeParameters(self):
                     for obj in self.parmConfObjList:
                         del obj
                     self.parmConfObjList = []
                 def addParameter(self, name, value, format='str'):
                     id = self.__getNewId()
                     parmConfObj = ParameterConf()
                     if not parmConfObj.setup(id, name, value, format):
                         return None
                     self.parmConfObjList.append(parmConfObj)
                     return parmConfObj
                 def changeParameter(self, name, value, format='str'):
                     parmConfObj = self.getParameterObj(name)
                     parmConfObj.update(name, value, format)
                     return parmConfObj
                 def makeXml(self, procUnitElement):
                     opElement = SubElement(procUnitElement, self.ELEMENTNAME)
                     opElement.set('id', str(self.id))
                     opElement.set('name', self.name)
                     opElement.set('type', self.type)
                     opElement.set('priority', str(self.priority))
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.makeXml(opElement)
                 def readXml(self, opElement):
                     self.id = opElement.get('id')
                     self.name = opElement.get('name')
                     self.type = opElement.get('type')
                     self.priority = opElement.get('priority')
                     #Compatible with old signal chain version
                     #Use of 'run' method instead 'init'
                     if self.type == 'self' and self.name == 'init':
                         self.name = 'run'
                     self.parmConfObjList = []
                     parmElementList = opElement.iter(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         #Compatible with old signal chain version
                         #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
                         if self.type != 'self' and self.name == 'Plot':
                             if parmConfObj.format == 'str' and parmConfObj.name == 'type':
                                 self.name = parmConfObj.value
                                 continue
                         self.parmConfObjList.append(parmConfObj)
                 def printattr(self):
                     print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
                                                                           self.id,
                                                                           self.name,
                                                                           self.type,
                                                                           self.priority)
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.printattr()
                 def createObject(self, plotter_queue=None):
                     if self.type == 'self':
                         raise ValueError, "This operation type cannot be created"
                     if self.type == 'plotter':
                         #Plotter(plotter_name)
                         if not plotter_queue:
                             raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
                         opObj = Plotter(self.name, plotter_queue)
                     if self.type == 'external' or self.type == 'other':
                         className = eval(self.name)
                         kwargs = self.getKwargs()
                         opObj = className(**kwargs)
                     return opObj
             class ProcUnitConf():
                 id = None
                 name = None
                 datatype = None
                 inputId = None
                 parentId = None
                 opConfObjList = []
                 procUnitObj = None
                 opObjList = []
                 ELEMENTNAME = 'ProcUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.opConfObjList = []
                     self.procUnitObj = None
                     self.opObjDict = {}
                 def __getPriority(self):
                     return len(self.opConfObjList)+1
                 def __getNewId(self):
                     return int(self.id)*10 + len(self.opConfObjList) + 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def updateId(self, new_id, parentId=parentId):
                     new_id = int(parentId)*10 + (int(self.id) % 10)
                     new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
                     #If this proc unit has not inputs
                     if self.inputId == '0':
                         new_inputId = 0
                     n = 1
                     for opConfObj in self.opConfObjList:
                         idOp = str(int(new_id)*10 + n)
                         opConfObj.updateId(idOp)
                         n += 1
                     self.parentId = str(parentId)
                     self.id = str(new_id)
                     self.inputId = str(new_inputId)
                 def getInputId(self):
                     return self.inputId
                 def getOperationObjList(self):
                     return self.opConfObjList
                 def getOperationObj(self, name=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.name != name:
                             continue
                         return opConfObj
                     return None
                 def getOpObjfromParamValue(self, value=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
                             continue
                         return opConfObj
                     return None
                 def getProcUnitObj(self):
                     return self.procUnitObj
                 def setup(self, id, name, datatype, inputId, parentId=None):
                     #Compatible with old signal chain version
                     if datatype==None and name==None:
                         raise ValueError, "datatype or name should be defined"
                     if name==None:
                         if 'Proc' in datatype:
                             name = datatype
                         else:
                             name = '%sProc' %(datatype)
                     if datatype==None:
                         datatype = name.replace('Proc','')
                     self.id = str(id)
                     self.name = name
                     self.datatype = datatype
                     self.inputId = inputId
                     self.parentId = parentId
                     self.opConfObjList = []
                     self.addOperation(name='run', optype='self')
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                     self.addOperation(name='run')
                 def addParameter(self, **kwargs):
                     '''
                     Add parameters to "run" operation
                     '''
                     opObj = self.opConfObjList[0]
                     opObj.addParameter(**kwargs)
                     return opObj
                 def addOperation(self, name, optype='self'):
                     id = self.__getNewId()
                     priority = self.__getPriority()
                     opConfObj = OperationConf()
                     opConfObj.setup(id, name=name, priority=priority, type=optype)
                     self.opConfObjList.append(opConfObj)
                     return opConfObj
                 def makeXml(self, projectElement):
                     procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
                     procUnitElement.set('id', str(self.id))
                     procUnitElement.set('name', self.name)
                     procUnitElement.set('datatype', self.datatype)
                     procUnitElement.set('inputId', str(self.inputId))
                     for opConfObj in self.opConfObjList:
                         opConfObj.makeXml(procUnitElement)
                 def readXml(self, upElement):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.inputId = upElement.get('inputId')
                     if self.ELEMENTNAME == "ReadUnit":
                         self.datatype = self.datatype.replace("Reader", "")
                     if self.ELEMENTNAME == "ProcUnit":
                         self.datatype = self.datatype.replace("Proc", "")
                     if self.inputId == 'None':
                         self.inputId = '0'
                     self.opConfObjList = []
                     opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement)
                         self.opConfObjList.append(opConfObj)
                 def printattr(self):
                     print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
                                                                          self.id,
                                                                          self.name,
                                                                          self.datatype,
                                                                          self.inputId)
                     for opConfObj in self.opConfObjList:
                         opConfObj.printattr()
                 def getKwargs(self):
                     opObj = self.opConfObjList[0]
                     kwargs = opObj.getKwargs()
                     return kwargs
                 def createObjects(self, plotter_queue=None):
                     className = eval(self.name)
                     kwargs = self.getKwargs()
                     procUnitObj = className(**kwargs)
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type=='self' and self.name=='run':
                             continue
                         elif opConfObj.type=='self':
                             procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
                             continue
                         opObj = opConfObj.createObject(plotter_queue)
                         self.opObjDict[opConfObj.id] = opObj
                         procUnitObj.addOperation(opObj, opConfObj.id)
                     self.procUnitObj = procUnitObj
                     return procUnitObj
                 def run(self):
                     is_ok = False
                     for opConfObj in self.opConfObjList:
                         kwargs = {}
                         for parmConfObj in opConfObj.getParameterObjList():
                             if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
                                 continue
                             kwargs[parmConfObj.name] = parmConfObj.getValue()
                         #ini = time.time()
                         #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
                         sts = self.procUnitObj.call(opType = opConfObj.type,
                                                     opName = opConfObj.name,
                                                     opId = opConfObj.id,
                                                     )
             #             total_time = time.time() - ini
             #
             #             if total_time > 0.002:
             #                 print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
                         is_ok = is_ok or sts
                     return is_ok
                 def close(self):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self':
                             continue
                         opObj = self.procUnitObj.getOperationObj(opConfObj.id)
                         opObj.close()
                     self.procUnitObj.close()
                     return
             class ReadUnitConf(ProcUnitConf):
                 path = None
                 startDate = None
                 endDate = None
                 startTime = None
                 endTime = None
                 ELEMENTNAME = 'ReadUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.parentId = None
                     self.opConfObjList = []
                     self.opObjList = []
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, queue=None, **kwargs):
                     #Compatible with old signal chain version
                     if datatype==None and name==None:
                         raise ValueError, "datatype or name should be defined"
                     if name==None:
                         if 'Reader' in datatype:
                             name = datatype
                         else:
                             name = '%sReader' %(datatype)
                     if datatype==None:
                         datatype = name.replace('Reader','')
                     self.id = id
                     self.name = name
                     self.datatype = datatype
                     self.path = os.path.abspath(path)
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.inputId = '0'
                     self.parentId = parentId
                     self.queue = queue
                     self.addRunOperation(**kwargs)
                 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
                     #Compatible with old signal chain version
                     if datatype==None and name==None:
                         raise ValueError, "datatype or name should be defined"
                     if name==None:
                         if 'Reader' in datatype:
                             name = datatype
                         else:
                             name = '%sReader' %(datatype)
                     if datatype==None:
                         datatype = name.replace('Reader','')
                     self.datatype = datatype
                     self.name = name
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.inputId = '0'
                     self.parentId = parentId
                     self.updateRunOperation(**kwargs)
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                 def addRunOperation(self, **kwargs):
                     opObj = self.addOperation(name = 'run', optype = 'self')
                     opObj.addParameter(name='datatype' , value=self.datatype, format='str')
                     opObj.addParameter(name='path'     , value=self.path, format='str')
                     opObj.addParameter(name='startDate' , value=self.startDate, format='date')
                     opObj.addParameter(name='endDate'   , value=self.endDate, format='date')
                     opObj.addParameter(name='startTime' , value=self.startTime, format='time')
                     opObj.addParameter(name='endTime'   , value=self.endTime, format='time')
                     opObj.addParameter(name='queue'   , value=self.queue, format='obj')
                     for key, value in kwargs.items():
                         opObj.addParameter(name=key, value=value, format=type(value).__name__)
                     return opObj
                 def updateRunOperation(self, **kwargs):
                     opObj = self.getOperationObj(name = 'run')
                     opObj.removeParameters()
                     opObj.addParameter(name='datatype' , value=self.datatype, format='str')
                     opObj.addParameter(name='path'     , value=self.path, format='str')
                     opObj.addParameter(name='startDate' , value=self.startDate, format='date')
                     opObj.addParameter(name='endDate'   , value=self.endDate, format='date')
                     opObj.addParameter(name='startTime' , value=self.startTime, format='time')
                     opObj.addParameter(name='endTime'   , value=self.endTime, format='time')
                     for key, value in kwargs.items():
                         opObj.addParameter(name=key, value=value, format=type(value).__name__)
                     return opObj
             #     def makeXml(self, projectElement):
             #
             #         procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
             #         procUnitElement.set('id', str(self.id))
             #         procUnitElement.set('name', self.name)
             #         procUnitElement.set('datatype', self.datatype)
             #         procUnitElement.set('inputId', str(self.inputId))
             #
             #         for opConfObj in self.opConfObjList:
             #             opConfObj.makeXml(procUnitElement)
                 def readXml(self, upElement):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.inputId = upElement.get('inputId')
                     if self.ELEMENTNAME == "ReadUnit":
                         self.datatype = self.datatype.replace("Reader", "")
                     if self.inputId == 'None':
                         self.inputId = '0'
                     self.opConfObjList = []
                     opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement)
                         self.opConfObjList.append(opConfObj)
                         if opConfObj.name == 'run':
                             self.path = opConfObj.getParameterValue('path')
                             self.startDate = opConfObj.getParameterValue('startDate')
                             self.endDate = opConfObj.getParameterValue('endDate')
                             self.startTime = opConfObj.getParameterValue('startTime')
                             self.endTime = opConfObj.getParameterValue('endTime')
             class Project():
                 id = None
                 name = None
                 description = None
                 filename = None
                 procUnitConfObjDict = None
                 ELEMENTNAME = 'Project'
                 plotterQueue = None
                 def __init__(self, plotter_queue=None):
                     self.id = None
                     self.name = None
                     self.description = None
                     self.plotterQueue = plotter_queue
                     self.procUnitConfObjDict = {}
                 def __getNewId(self):
                     idList = self.procUnitConfObjDict.keys()
                     id = int(self.id)*10
                     while True:
                         id += 1
                         if str(id) in idList:
                             continue
                         break
                     return str(id)
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     keyList = self.procUnitConfObjDict.keys()
                     keyList.sort()
                     n = 1
                     newProcUnitConfObjDict = {}
                     for procKey in keyList:
                         procUnitConfObj = self.procUnitConfObjDict[procKey]
                         idProcUnit = str(int(self.id)*10 + n)
                         procUnitConfObj.updateId(idProcUnit, parentId = self.id)
                         newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
                         n += 1
                     self.procUnitConfObjDict = newProcUnitConfObjDict
                 def setup(self, id, name, description):
                     self.id = str(id)
                     self.name = name
                     self.description = description
                 def update(self, name, description):
                     self.name = name
                     self.description = description
                 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
                     if id is None:
                         idReadUnit = self.__getNewId()
                     else:
                         idReadUnit = str(id)
                     readUnitConfObj = ReadUnitConf()
                     readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
                     self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     return readUnitConfObj
                 def addProcUnit(self, inputId='0', datatype=None, name=None):
                     idProcUnit = self.__getNewId()
                     procUnitConfObj = ProcUnitConf()
                     procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
                     self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     return procUnitConfObj
                 def removeProcUnit(self, id):
                     if id in self.procUnitConfObjDict.keys():
                         self.procUnitConfObjDict.pop(id)
                 def getReadUnitId(self):
                     readUnitConfObj = self.getReadUnitObj()
                     return readUnitConfObj.id
                 def getReadUnitObj(self):
                     for obj in self.procUnitConfObjDict.values():
                         if obj.getElementName() == "ReadUnit":
                             return obj
                     return None
                 def getProcUnitObj(self, id=None, name=None):
                     if id != None:
                         return self.procUnitConfObjDict[id]
                     if name != None:
                         return self.getProcUnitObjByName(name)
                     return None
                 def getProcUnitObjByName(self, name):
                     for obj in self.procUnitConfObjDict.values():
                         if obj.name == name:
                             return obj
                     return None
                 def procUnitItems(self):
                     return self.procUnitConfObjDict.items()
                 def makeXml(self):
                     projectElement = Element('Project')
                     projectElement.set('id', str(self.id))
                     projectElement.set('name', self.name)
                     projectElement.set('description', self.description)
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.makeXml(projectElement)
                     self.projectElement = projectElement
                 def writeXml(self, filename=None):
                     if filename == None:
                         if self.filename:
                             filename = self.filename
                         else:
                             filename = "schain.xml"
                     if not filename:
                         print "filename has not been defined. Use setFilename(filename) for do it."
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.access(os.path.dirname(abs_file), os.W_OK):
                         print "No write permission on %s" %os.path.dirname(abs_file)
                         return 0
                     if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
                         print "File %s already exists and it could not be overwriten" %abs_file
                         return 0
                     self.makeXml()
                     ElementTree(self.projectElement).write(abs_file, method='xml')
                     self.filename = abs_file
                     return 1
                 def readXml(self, filename = None):
                     if not filename:
                         print "filename is not defined"
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.path.isfile(abs_file):
                         print "%s file does not exist" %abs_file
                         return 0
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     try:
                         self.projectElement = ElementTree().parse(abs_file)
                     except:
                         print "Error reading %s, verify file format" %filename
                         return 0
                     self.project = self.projectElement.tag
                     self.id = self.projectElement.get('id')
                     self.name = self.projectElement.get('name')
                     self.description = self.projectElement.get('description')
                     readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
                     for readUnitElement in readUnitElementList:
                         readUnitConfObj = ReadUnitConf()
                         readUnitConfObj.readXml(readUnitElement)
                         if readUnitConfObj.parentId == None:
                             readUnitConfObj.parentId = self.id
                         self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
                     for procUnitElement in procUnitElementList:
                         procUnitConfObj = ProcUnitConf()
                         procUnitConfObj.readXml(procUnitElement)
                         if procUnitConfObj.parentId == None:
                             procUnitConfObj.parentId = self.id
                         self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     self.filename = abs_file
                     return 1
                 def printattr(self):
                     print "Project[%s]: name = %s, description = %s" %(self.id,
                                                                           self.name,
                                                                           self.description)
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.printattr()
                 def createObjects(self):
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         procUnitConfObj.createObjects(self.plotterQueue)
                 def __connect(self, objIN, thisObj):
                     thisObj.setInput(objIN.getOutputObj())
                 def connectObjects(self):
                     for thisPUConfObj in self.procUnitConfObjDict.values():
                         inputId = thisPUConfObj.getInputId()
                         if int(inputId) == 0:
                             continue
                         #Get input object
                         puConfINObj = self.procUnitConfObjDict[inputId]
                         puObjIN = puConfINObj.getProcUnitObj()
                         #Get current object
                         thisPUObj = thisPUConfObj.getProcUnitObj()
                         self.__connect(puObjIN, thisPUObj)
                 def __handleError(self, procUnitConfObj, send_email=True):
                     import socket
                     err = traceback.format_exception(sys.exc_info()[0],
                                                      sys.exc_info()[1],
                                                      sys.exc_info()[2])
                     print "***** Error occurred in %s *****" %(procUnitConfObj.name)
                     print "***** %s" %err[-1]
                     message = "".join(err)
                     sys.stderr.write(message)
                     if not send_email:
                         return
                     subject =  "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
                     subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
                     subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
                     subtitle += "Working directory: %s\n" %os.path.abspath("./")
                     subtitle += "Configuration file: %s\n" %self.filename
                     subtitle += "Time: %s\n" %str(datetime.datetime.now())
                     readUnitConfObj = self.getReadUnitObj()
                     if readUnitConfObj:
                         subtitle += "\nInput parameters:\n"
                         subtitle += "[Data path = %s]\n" %readUnitConfObj.path
                         subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
                         subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
                         subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
                         subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
                         subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
                     adminObj = schainpy.admin.SchainNotify()
                     adminObj.sendAlert(message=message,
                                        subject=subject,
                                        subtitle=subtitle,
                                        filename=self.filename)
                 def isPaused(self):
                     return 0
                 def isStopped(self):
                     return 0
                 def runController(self):
                     """
                     returns 0 when this process has been stopped, 1 otherwise
                     """
                     if self.isPaused():
                         print "Process suspended"
                         while True:
                             sleep(0.1)
                             if not self.isPaused():
                                 break
                             if self.isStopped():
                                 break
                         print "Process reinitialized"
                     if self.isStopped():
                         print "Process stopped"
                         return 0
                     return 1
                 def setFilename(self, filename):
                     self.filename = filename
                 def setPlotterQueue(self, plotter_queue):
                     raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
                 def getPlotterQueue(self):
                     raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
                 def useExternalPlotter(self):
                     raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
                 def run(self):
                     print
                     print "*"*60
                     print "   Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
                     print "*"*60
                     print
                     keyList = self.procUnitConfObjDict.keys()
                     keyList.sort()
                     while(True):
                         is_ok = False
                         for procKey in keyList:
             #                 print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
                             procUnitConfObj = self.procUnitConfObjDict[procKey]
                             try:
                                 sts = procUnitConfObj.run()
                                 is_ok = is_ok or sts
                             except KeyboardInterrupt:
                                 is_ok = False
                                 break
                             except ValueError, e:
                                 sleep(0.5)
                                 self.__handleError(procUnitConfObj, send_email=True)
                                 is_ok = False
                                 break
                             except:
                                 sleep(0.5)
                                 self.__handleError(procUnitConfObj)
                                 is_ok = False
                                 break
                         #If every process unit finished so end process
                         if not(is_ok):
             #                 print "Every process unit have finished"
                             break
                         if not self.runController():
                             break
                     #Closing every process
                     for procKey in keyList:
                         procUnitConfObj = self.procUnitConfObjDict[procKey]
                         procUnitConfObj.close()
                     print "Process finished"
                 def start(self):
                     self.writeXml()
                     self.createObjects()
                     self.connectObjects()
                     self.run()

schainpy/model/data/jrodata.py +1 -1

             '''
             $Author: murco $
             $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
             '''
             import copy
             import numpy
             import datetime
             from jroheaderIO import SystemHeader, RadarControllerHeader
             from schainpy import cSchain
             def getNumpyDtype(dataTypeCode):
                 if dataTypeCode == 0:
                     numpyDtype = numpy.dtype([('real','<i1'),('imag','<i1')])
                 elif dataTypeCode == 1:
                     numpyDtype = numpy.dtype([('real','<i2'),('imag','<i2')])
                 elif dataTypeCode == 2:
                     numpyDtype = numpy.dtype([('real','<i4'),('imag','<i4')])
                 elif dataTypeCode == 3:
                     numpyDtype = numpy.dtype([('real','<i8'),('imag','<i8')])
                 elif dataTypeCode == 4:
                     numpyDtype = numpy.dtype([('real','<f4'),('imag','<f4')])
                 elif dataTypeCode == 5:
                     numpyDtype = numpy.dtype([('real','<f8'),('imag','<f8')])
                 else:
                     raise ValueError, 'dataTypeCode was not defined'
                 return numpyDtype
             def getDataTypeCode(numpyDtype):
                 if numpyDtype == numpy.dtype([('real','<i1'),('imag','<i1')]):
                     datatype = 0
                 elif numpyDtype == numpy.dtype([('real','<i2'),('imag','<i2')]):
                     datatype = 1
                 elif numpyDtype == numpy.dtype([('real','<i4'),('imag','<i4')]):
                     datatype = 2
                 elif numpyDtype == numpy.dtype([('real','<i8'),('imag','<i8')]):
                     datatype = 3
                 elif numpyDtype == numpy.dtype([('real','<f4'),('imag','<f4')]):
                     datatype = 4
                 elif numpyDtype == numpy.dtype([('real','<f8'),('imag','<f8')]):
                     datatype = 5
                 else:
                     datatype = None
                 return datatype
             def hildebrand_sekhon(data, navg):
                 """
                 This method is for the objective determination of the noise level in Doppler spectra. This
                 implementation technique is based on the fact that the standard deviation of the spectral
                 densities is equal to the mean spectral density for white Gaussian noise
                 Inputs:
                     Data    :    heights
                     navg    :    numbers of averages
                 Return:
                     -1        :    any error
                     anoise    :    noise's level
                 """
-                sortdata = numpy.sort(data,axis=None)
+                sortdata = numpy.sort(data, axis=None)
             #     lenOfData = len(sortdata)
             #     nums_min = lenOfData*0.2
             #
             #     if nums_min <= 5:
             #         nums_min = 5
             #
             #     sump = 0.
             #
             #     sumq = 0.
             #
             #     j = 0
             #
             #     cont = 1
             #
             #     while((cont==1)and(j<lenOfData)):
             #
             #         sump += sortdata[j]
             #
             #         sumq += sortdata[j]**2
             #
             #         if j > nums_min:
             #             rtest = float(j)/(j-1) + 1.0/navg
             #             if ((sumq*j) > (rtest*sump**2)):
             #                 j = j - 1
             #                 sump  = sump - sortdata[j]
             #                 sumq =  sumq - sortdata[j]**2
             #                 cont = 0
             #
             #         j += 1
             #
             #     lnoise = sump /j
             #
             #     return lnoise
                 return cSchain.hildebrand_sekhon(sortdata, navg)
             class Beam:
                 def __init__(self):
                     self.codeList = []
                     self.azimuthList = []
                     self.zenithList = []
             class GenericData(object):
                 flagNoData = True
                 def __init__(self):
                     raise NotImplementedError
                 def copy(self, inputObj=None):
                     if inputObj == None:
                         return copy.deepcopy(self)
                     for key in inputObj.__dict__.keys():
                         attribute = inputObj.__dict__[key]
                         #If this attribute is a tuple or list
                         if type(inputObj.__dict__[key]) in (tuple, list):
                             self.__dict__[key] = attribute[:]
                             continue
                         #If this attribute is another object or instance
                         if hasattr(attribute, '__dict__'):
                             self.__dict__[key] = attribute.copy()
                             continue
                         self.__dict__[key] = inputObj.__dict__[key]
                 def deepcopy(self):
                     return copy.deepcopy(self)
                 def isEmpty(self):
                     return self.flagNoData
             class JROData(GenericData):
             #    m_BasicHeader = BasicHeader()
             #    m_ProcessingHeader = ProcessingHeader()
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
             #    data = None
                 type = None
                 datatype = None #dtype but in string
             #     dtype = None
             #    nChannels = None
             #    nHeights = None
                 nProfiles = None
                 heightList = None
                 channelList = None
                 flagDiscontinuousBlock = False
                 useLocalTime = False
                 utctime = None
                 timeZone = None
                 dstFlag = None
                 errorCount = None
                 blocksize = None
             #     nCode = None
             #
             #     nBaud = None
             #
             #     code = None
                 flagDecodeData = False #asumo q la data no esta decodificada
                 flagDeflipData = False #asumo q la data no esta sin flip
                 flagShiftFFT = False
             #     ippSeconds = None
             #     timeInterval = None
                 nCohInt = None
             #     noise = None
                 windowOfFilter = 1
                 #Speed of ligth
                 C = 3e8
                 frequency = 49.92e6
                 realtime = False
                 beacon_heiIndexList = None
                 last_block = None
                 blocknow = None
                 azimuth = None
                 zenith = None
                 beam = Beam()
                 profileIndex = None
                 def __init__(self):
                     raise NotImplementedError
                 def getNoise(self):
                     raise NotImplementedError
                 def getNChannels(self):
                     return len(self.channelList)
                 def getChannelIndexList(self):
                     return range(self.nChannels)
                 def getNHeights(self):
                     return len(self.heightList)
                 def getHeiRange(self, extrapoints=0):
                     heis = self.heightList
             #        deltah = self.heightList[1] - self.heightList[0]
             #
             #        heis.append(self.heightList[-1])
                     return heis
                 def getDeltaH(self):
                     delta = self.heightList[1] - self.heightList[0]
                     return delta
                 def getltctime(self):
                     if self.useLocalTime:
                         return self.utctime - self.timeZone*60
                     return self.utctime
                 def getDatatime(self):
                     datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
                     return datatimeValue
                 def getTimeRange(self):
                     datatime = []
                     datatime.append(self.ltctime)
                     datatime.append(self.ltctime + self.timeInterval+1)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getFmaxTimeResponse(self):
                     period = (10**-6)*self.getDeltaH()/(0.15)
                     PRF = 1./(period * self.nCohInt)
                     fmax = PRF
                     return fmax
                 def getFmax(self):
                     PRF = 1./(self.ippSeconds * self.nCohInt)
                     fmax = PRF
                     return fmax
                 def getVmax(self):
                     _lambda = self.C/self.frequency
                     vmax = self.getFmax() * _lambda/2
                     return vmax
                 def get_ippSeconds(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.ippSeconds
                 def set_ippSeconds(self, ippSeconds):
                     '''
                     '''
                     self.radarControllerHeaderObj.ippSeconds = ippSeconds
                     return
                 def get_dtype(self):
                     '''
                     '''
                     return getNumpyDtype(self.datatype)
                 def set_dtype(self, numpyDtype):
                     '''
                     '''
                     self.datatype = getDataTypeCode(numpyDtype)
                 def get_code(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.code
                 def set_code(self, code):
                     '''
                     '''
                     self.radarControllerHeaderObj.code = code
                     return
                 def get_ncode(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.nCode
                 def set_ncode(self, nCode):
                     '''
                     '''
                     self.radarControllerHeaderObj.nCode = nCode
                     return
                 def get_nbaud(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.nBaud
                 def set_nbaud(self, nBaud):
                     '''
                     '''
                     self.radarControllerHeaderObj.nBaud = nBaud
                     return
                 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
                 channelIndexList = property(getChannelIndexList, "I'm the 'channelIndexList' property.")
                 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
                 #noise = property(getNoise, "I'm the 'nHeights' property.")
                 datatime = property(getDatatime, "I'm the 'datatime' property")
                 ltctime = property(getltctime, "I'm the 'ltctime' property")
                 ippSeconds = property(get_ippSeconds, set_ippSeconds)
                 dtype = property(get_dtype, set_dtype)
             #     timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
                 code = property(get_code, set_code)
                 nCode = property(get_ncode, set_ncode)
                 nBaud = property(get_nbaud, set_nbaud)
             class Voltage(JROData):
                 #data es un numpy array de 2 dmensiones (canales, alturas)
                 data = None
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.useLocalTime = True
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Voltage"
                     self.data = None
             #         self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
                     self.timeZone = None
                     self.dstFlag = None
                     self.errorCount = None
                     self.nCohInt = None
                     self.blocksize = None
                     self.flagDecodeData = False #asumo q la data no esta decodificada
                     self.flagDeflipData = False #asumo q la data no esta sin flip
                     self.flagShiftFFT = False
                     self.flagDataAsBlock = False    #Asumo que la data es leida perfil a perfil
                     self.profileIndex = 0
                 def getNoisebyHildebrand(self, channel = None):
                     """
                     Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                     Return:
                         noiselevel
                     """
                     if channel != None:
                         data = self.data[channel]
                         nChannels = 1
                     else:
                         data = self.data
                         nChannels = self.nChannels
                     noise = numpy.zeros(nChannels)
                     power = data * numpy.conjugate(data)
                     for thisChannel in range(nChannels):
                         if nChannels == 1:
                             daux = power[:].real
                         else:
                             daux = power[thisChannel,:].real
                         noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
                     return noise
                 def getNoise(self, type = 1, channel = None):
                     if type == 1:
                         noise = self.getNoisebyHildebrand(channel)
                     return noise
                 def getPower(self, channel = None):
                     if channel != None:
                         data = self.data[channel]
                     else:
                         data = self.data
                     power = data * numpy.conjugate(data)
                     powerdB = 10*numpy.log10(power.real)
                     powerdB = numpy.squeeze(powerdB)
                     return powerdB
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt
                     return timeInterval
                 noise = property(getNoise, "I'm the 'nHeights' property.")
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
             class Spectra(JROData):
                 #data spc es un numpy array de 2 dmensiones (canales, perfiles, alturas)
                 data_spc = None
                 #data cspc es un numpy array de 2 dmensiones (canales, pares, alturas)
                 data_cspc = None
                 #data dc es un numpy array de 2 dmensiones (canales, alturas)
                 data_dc = None
                 #data power
                 data_pwr = None
                 nFFTPoints = None
             #     nPairs = None
                 pairsList = None
                 nIncohInt = None
                 wavelength = None #Necesario para cacular el rango de velocidad desde la frecuencia
                 nCohInt = None #se requiere para determinar el valor de timeInterval
                 ippFactor = None
                 profileIndex = 0
                 plotting = "spectra"
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.useLocalTime = True
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Spectra"
             #        self.data = None
             #         self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.pairsList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
                     self.nCohInt = None
                     self.nIncohInt = None
                     self.blocksize = None
                     self.nFFTPoints = None
                     self.wavelength = None
                     self.flagDecodeData = False #asumo q la data no esta decodificada
                     self.flagDeflipData = False #asumo q la data no esta sin flip
                     self.flagShiftFFT = False
                     self.ippFactor = 1
                     #self.noise = None
                     self.beacon_heiIndexList = []
                     self.noise_estimation = None
                 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
                     """
                     Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                     Return:
                         noiselevel
                     """
                     noise = numpy.zeros(self.nChannels)
                     for channel in range(self.nChannels):
                         daux = self.data_spc[channel,xmin_index:xmax_index,ymin_index:ymax_index]
                         noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
                     return noise
                 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
                     if self.noise_estimation is not None:
                         return self.noise_estimation #this was estimated by getNoise Operation defined in jroproc_spectra.py
                     else:
                         noise = self.getNoisebyHildebrand(xmin_index, xmax_index, ymin_index, ymax_index)
                         return noise
                 def getFreqRangeTimeResponse(self, extrapoints=0):
                     deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints*self.ippFactor)
                     freqrange = deltafreq*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltafreq/2
                     return freqrange
                 def getAcfRange(self, extrapoints=0):
                     deltafreq = 10./(self.getFmax() / (self.nFFTPoints*self.ippFactor))
                     freqrange = deltafreq*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltafreq/2
                     return freqrange
                 def getFreqRange(self, extrapoints=0):
                     deltafreq = self.getFmax() / (self.nFFTPoints*self.ippFactor)
                     freqrange = deltafreq*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltafreq/2
                     return freqrange
                 def getVelRange(self, extrapoints=0):
                     deltav = self.getVmax() / (self.nFFTPoints*self.ippFactor)
                     velrange = deltav*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) #- deltav/2
                     return velrange
                 def getNPairs(self):
                     return len(self.pairsList)
                 def getPairsIndexList(self):
                     return range(self.nPairs)
                 def getNormFactor(self):
                     pwcode = 1
                     if self.flagDecodeData:
                         pwcode = numpy.sum(self.code[0]**2)
                     #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
                     normFactor = self.nProfiles*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
                     return normFactor
                 def getFlagCspc(self):
                     if self.data_cspc is None:
                         return True
                     return False
                 def getFlagDc(self):
                     if self.data_dc is None:
                         return True
                     return False
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles
                     return timeInterval
                 def getPower(self):
                     factor = self.normFactor
                     z = self.data_spc/factor
                     z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                     avg = numpy.average(z, axis=1)
                     return 10*numpy.log10(avg)
                 def getCoherence(self, pairsList=None, phase=False):
                     z = []
                     if pairsList is None:
                         pairsIndexList = self.pairsIndexList
                     else:
                         pairsIndexList = []
                         for pair in pairsList:
                             if pair not in self.pairsList:
                                 raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
                             pairsIndexList.append(self.pairsList.index(pair))
                     for i in range(len(pairsIndexList)):
                         pair = self.pairsList[pairsIndexList[i]]
                         ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
                         powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
                         powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
                         avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
                         if phase:
                             data = numpy.arctan2(avgcoherenceComplex.imag,
                                                  avgcoherenceComplex.real)*180/numpy.pi
                         else:
                             data = numpy.abs(avgcoherenceComplex)
                         z.append(data)
                     return numpy.array(z)
                 def setValue(self, value):
                     print "This property should not be initialized"
                     return
                 nPairs = property(getNPairs, setValue, "I'm the 'nPairs' property.")
                 pairsIndexList = property(getPairsIndexList, setValue, "I'm the 'pairsIndexList' property.")
                 normFactor = property(getNormFactor, setValue, "I'm the 'getNormFactor' property.")
                 flag_cspc = property(getFlagCspc, setValue)
                 flag_dc = property(getFlagDc, setValue)
                 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
                 timeInterval = property(getTimeInterval, setValue, "I'm the 'timeInterval' property")
             class SpectraHeis(Spectra):
                 data_spc = None
                 data_cspc = None
                 data_dc = None
                 nFFTPoints = None
             #     nPairs = None
                 pairsList = None
                 nCohInt = None
                 nIncohInt = None
                 def __init__(self):
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "SpectraHeis"
             #         self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
             #         self.nPairs = 0
                     self.utctime = None
                     self.blocksize = None
                     self.profileIndex = 0
                     self.nCohInt = 1
                     self.nIncohInt = 1
                 def getNormFactor(self):
                     pwcode = 1
                     if self.flagDecodeData:
                         pwcode = numpy.sum(self.code[0]**2)
                     normFactor = self.nIncohInt*self.nCohInt*pwcode
                     return normFactor
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     return timeInterval
                 normFactor = property(getNormFactor, "I'm the 'getNormFactor' property.")
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
             class Fits(JROData):
                 heightList = None
                 channelList = None
                 flagNoData = True
                 flagDiscontinuousBlock = False
                 useLocalTime = False
                 utctime = None
                 timeZone = None
             #     ippSeconds = None
             #     timeInterval = None
                 nCohInt = None
                 nIncohInt = None
                 noise = None
                 windowOfFilter = 1
                 #Speed of ligth
                 C = 3e8
                 frequency = 49.92e6
                 realtime = False
                 def __init__(self):
                     self.type = "Fits"
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #         self.channelIndexList = None
                     self.flagNoData = True
                     self.utctime = None
                     self.nCohInt = 1
                     self.nIncohInt = 1
                     self.useLocalTime = True
                     self.profileIndex = 0
             #         self.utctime = None
             #         self.timeZone = None
             #         self.ltctime = None
             #         self.timeInterval = None
             #         self.header = None
             #         self.data_header = None
             #         self.data = None
             #         self.datatime = None
             #         self.flagNoData = False
             #         self.expName = ''
             #         self.nChannels = None
             #         self.nSamples = None
             #         self.dataBlocksPerFile = None
             #         self.comments = ''
             #
                 def getltctime(self):
                     if self.useLocalTime:
                         return self.utctime - self.timeZone*60
                     return self.utctime
                 def getDatatime(self):
                     datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
                     return datatime
                 def getTimeRange(self):
                     datatime = []
                     datatime.append(self.ltctime)
                     datatime.append(self.ltctime + self.timeInterval)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getHeiRange(self):
                     heis = self.heightList
                     return heis
                 def getNHeights(self):
                     return len(self.heightList)
                 def getNChannels(self):
                     return len(self.channelList)
                 def getChannelIndexList(self):
                     return range(self.nChannels)
                 def getNoise(self, type = 1):
                     #noise = numpy.zeros(self.nChannels)
                     if type == 1:
                         noise = self.getNoisebyHildebrand()
                     if type == 2:
                         noise = self.getNoisebySort()
                     if type == 3:
                         noise = self.getNoisebyWindow()
                     return noise
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     return timeInterval
                 datatime = property(getDatatime, "I'm the 'datatime' property")
                 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
                 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
                 channelIndexList = property(getChannelIndexList, "I'm the 'channelIndexList' property.")
                 noise = property(getNoise, "I'm the 'nHeights' property.")
                 ltctime = property(getltctime, "I'm the 'ltctime' property")
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
             class Correlation(JROData):
                 noise = None
                 SNR = None
                 #--------------------------------------------------
                 mode = None
                 split = False
                 data_cf = None
                 lags = None
                 lagRange = None
                 pairsList = None
                 normFactor = None
                 #--------------------------------------------------
             #     calculateVelocity = None
                 nLags = None
                 nPairs = None
                 nAvg = None
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Correlation"
                     self.data = None
                     self.dtype = None
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
                     self.timeZone = None
                     self.dstFlag = None
                     self.errorCount = None
                     self.blocksize = None
                     self.flagDecodeData = False #asumo q la data no esta decodificada
                     self.flagDeflipData = False #asumo q la data no esta sin flip
                     self.pairsList = None
                     self.nPoints = None
                 def getPairsList(self):
                     return self.pairsList
                 def getNoise(self, mode = 2):
                     indR = numpy.where(self.lagR == 0)[0][0]
                     indT = numpy.where(self.lagT == 0)[0][0]
                     jspectra0 = self.data_corr[:,:,indR,:]
                     jspectra = copy.copy(jspectra0)
                     num_chan = jspectra.shape[0]
                     num_hei = jspectra.shape[2]
                     freq_dc = jspectra.shape[1]/2
                     ind_vel = numpy.array([-2,-1,1,2]) + freq_dc
                     if ind_vel[0]<0:
                         ind_vel[range(0,1)] = ind_vel[range(0,1)] + self.num_prof
                     if mode == 1:
                         jspectra[:,freq_dc,:] = (jspectra[:,ind_vel[1],:] + jspectra[:,ind_vel[2],:])/2 #CORRECCION
                     if mode == 2:
                         vel = numpy.array([-2,-1,1,2])
                         xx = numpy.zeros([4,4])
                         for fil in range(4):
                             xx[fil,:] = vel[fil]**numpy.asarray(range(4))
                         xx_inv = numpy.linalg.inv(xx)
                         xx_aux = xx_inv[0,:]
                         for ich in range(num_chan):
                             yy = jspectra[ich,ind_vel,:]
                             jspectra[ich,freq_dc,:] = numpy.dot(xx_aux,yy)
                             junkid = jspectra[ich,freq_dc,:]<=0
                             cjunkid = sum(junkid)
                             if cjunkid.any():
                                 jspectra[ich,freq_dc,junkid.nonzero()] = (jspectra[ich,ind_vel[1],junkid] + jspectra[ich,ind_vel[2],junkid])/2
                     noise = jspectra0[:,freq_dc,:] - jspectra[:,freq_dc,:]
                     return noise
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nProfiles
                     return timeInterval
                 def splitFunctions(self):
                     pairsList = self.pairsList
                     ccf_pairs = []
                     acf_pairs = []
                     ccf_ind = []
                     acf_ind = []
                     for l in range(len(pairsList)):
                         chan0 = pairsList[l][0]
                         chan1 = pairsList[l][1]
                         #Obteniendo pares de Autocorrelacion
                         if chan0 == chan1:
                             acf_pairs.append(chan0)
                             acf_ind.append(l)
                         else:
                             ccf_pairs.append(pairsList[l])
                             ccf_ind.append(l)
                     data_acf = self.data_cf[acf_ind]
                     data_ccf = self.data_cf[ccf_ind]
                     return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
                 def getNormFactor(self):
                     acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
                     acf_pairs = numpy.array(acf_pairs)
                     normFactor = numpy.zeros((self.nPairs,self.nHeights))
                     for p in range(self.nPairs):
                         pair = self.pairsList[p]
                         ch0 = pair[0]
                         ch1 = pair[1]
                         ch0_max = numpy.max(data_acf[acf_pairs==ch0,:,:], axis=1)
                         ch1_max = numpy.max(data_acf[acf_pairs==ch1,:,:], axis=1)
                         normFactor[p,:] = numpy.sqrt(ch0_max*ch1_max)
                     return normFactor
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
                 normFactor = property(getNormFactor, "I'm the 'normFactor property'")
             class Parameters(Spectra):
                 experimentInfo = None   #Information about the experiment
                 #Information from previous data
                 inputUnit = None        #Type of data to be processed
                 operation = None        #Type of operation to parametrize
                 #normFactor = None       #Normalization Factor
                 groupList = None        #List of Pairs, Groups, etc
                 #Parameters
                 data_param = None       #Parameters obtained
                 data_pre = None         #Data Pre Parametrization
                 data_SNR = None         #Signal to Noise Ratio
             #     heightRange = None      #Heights
                 abscissaList = None    #Abscissa, can be velocities, lags or time
             #    noise = None            #Noise Potency
                 utctimeInit = None      #Initial UTC time
                 paramInterval = None    #Time interval to calculate Parameters in seconds
                 useLocalTime = True
                 #Fitting
                 data_error = None       #Error of the estimation
                 constants = None
                 library = None
                 #Output signal
                 outputInterval = None     #Time interval to calculate output signal in seconds
                 data_output = None       #Out signal
                 nAvg = None
                 noise_estimation = None
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Parameters"
                 def getTimeRange1(self, interval):
                     datatime = []
                     if self.useLocalTime:
                         time1 = self.utctimeInit - self.timeZone*60
                     else:
                         time1 = self.utctimeInit
                     datatime.append(time1)
                     datatime.append(time1 + interval)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getTimeInterval(self):
                     return self.timeInterval1
                 def getNoise(self):
                     return self.spc_noise
                 timeInterval = property(getTimeInterval)

schainpy/model/graphics/jroplot_data.py +23 -9

             import os
             import zmq
             import time
             import numpy
             import datetime
             import numpy as np
             import matplotlib
             matplotlib.use('TkAgg')
             import matplotlib.pyplot as plt
             from mpl_toolkits.axes_grid1 import make_axes_locatable
             from matplotlib.ticker import FuncFormatter, LinearLocator
             from multiprocessing import Process
             from schainpy.model.proc.jroproc_base import Operation
             plt.ioff()
             func = lambda x, pos: ('%s') %(datetime.datetime.fromtimestamp(x).strftime('%H:%M'))
             d1970 = datetime.datetime(1970,1,1)
             class PlotData(Operation, Process):
                 CODE = 'Figure'
                 colormap = 'jro'
-                CONFLATE = True
+                CONFLATE = False
                 __MAXNUMX = 80
-                __MAXNUMY = 80
                 __missing = 1E30
                 def __init__(self, **kwargs):
                     Operation.__init__(self, plot=True, **kwargs)
                     Process.__init__(self)
                     self.kwargs['code'] = self.CODE
                     self.mp = False
                     self.dataOut = None
                     self.isConfig = False
                     self.figure = None
                     self.axes = []
                     self.localtime = kwargs.pop('localtime', True)
                     self.show = kwargs.get('show', True)
                     self.save = kwargs.get('save', False)
                     self.colormap = kwargs.get('colormap', self.colormap)
                     self.colormap_coh = kwargs.get('colormap_coh', 'jet')
                     self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
                     self.showprofile = kwargs.get('showprofile', True)
                     self.title = kwargs.get('wintitle', '')
                     self.xaxis = kwargs.get('xaxis', 'frequency')
                     self.zmin = kwargs.get('zmin', None)
                     self.zmax = kwargs.get('zmax', None)
                     self.xmin = kwargs.get('xmin', None)
                     self.xmax = kwargs.get('xmax', None)
                     self.xrange = kwargs.get('xrange', 24)
                     self.ymin = kwargs.get('ymin', None)
                     self.ymax = kwargs.get('ymax', None)
+                    self.__MAXNUMY = kwargs.get('decimation', 80)
                     self.throttle_value = 5
+                    self.times = []
                 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
                     if x_buffer.shape[0] < 2:
                         return x_buffer, y_buffer, z_buffer
                     deltas = x_buffer[1:] - x_buffer[0:-1]
                     x_median = np.median(deltas)
                     index = np.where(deltas > 5*x_median)
                     if len(index[0]) != 0:
                         z_buffer[::, index[0], ::] = self.__missing
                         z_buffer = np.ma.masked_inside(z_buffer,
 .99*self.__missing,
 .01*self.__missing)
                     return x_buffer, y_buffer, z_buffer
                 def decimate(self):
                     # dx = int(len(self.x)/self.__MAXNUMX) + 1
                     dy = int(len(self.y)/self.__MAXNUMY) + 1
                     # x = self.x[::dx]
                     x = self.x
                     y = self.y[::dy]
                     z = self.z[::, ::, ::dy]
                     return x, y, z
                 def __plot(self):
                     print 'plotting...{}'.format(self.CODE)
                     if self.show:
                         print 'showing'
                         self.figure.show()
                     self.plot()
                     plt.tight_layout()
                     self.figure.canvas.manager.set_window_title('{} {} - Date:{}'.format(self.title, self.CODE.upper(),
                                                                                          datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')))
                     if self.save:
                         figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,
                                                                              datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
                         print 'Saving figure: {}'.format(figname)
                         self.figure.savefig(figname)
                     self.figure.canvas.draw()
                 def plot(self):
                     print 'plotting...{}'.format(self.CODE.upper())
                     return
                 def run(self):
                     print '[Starting] {}'.format(self.name)
                     context = zmq.Context()
                     receiver = context.socket(zmq.SUB)
                     receiver.setsockopt(zmq.SUBSCRIBE, '')
                     receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
                     receiver.connect("ipc:///tmp/zmq.plots")
+                    seconds_passed = 0
                     while True:
                         try:
-                            self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
+                            self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)#flags=zmq.NOBLOCK
+                            self.started = self.data['STARTED']
                             self.dataOut = self.data['dataOut']
+                            if (len(self.times) < len(self.data['times']) and not self.started and self.data['ENDED']):
+                                continue
                             self.times = self.data['times']
                             self.times.sort()
                             self.throttle_value = self.data['throttle']
                             self.min_time = self.times[0]
                             self.max_time = self.times[-1]
                             if self.isConfig is False:
+                                print 'setting up'
                                 self.setup()
                                 self.isConfig = True
-                            self.__plot()
+                                self.__plot()
                             if self.data['ENDED'] is True:
+                                print '********GRAPHIC ENDED********'
+                                self.ended = True
                                 self.isConfig = False
+                                self.__plot()
+                            elif seconds_passed >= self.data['throttle']:
+                                print 'passed', seconds_passed
+                                self.__plot()
+                                seconds_passed = 0
                         except zmq.Again as e:
                             print 'Waiting for data...'
-                            plt.pause(self.throttle_value)
+                            plt.pause(2)
+                            seconds_passed += 2
                 def close(self):
                     if self.dataOut:
                         self.__plot()
             class PlotSpectraData(PlotData):
                 CODE = 'spc'
                 colormap = 'jro'
                 CONFLATE = False
                 def setup(self):
                     ncolspan = 1
                     colspan = 1
                     self.ncols = int(numpy.sqrt(self.dataOut.nChannels)+0.9)
                     self.nrows = int(self.dataOut.nChannels*1./self.ncols + 0.9)
                     self.width = 3.6*self.ncols
                     self.height = 3.2*self.nrows
                     if self.showprofile:
                         ncolspan = 3
                         colspan = 2
                         self.width += 1.2*self.ncols
                     self.ylabel = 'Range [Km]'
                     self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
                     if self.figure is None:
                         self.figure = plt.figure(figsize=(self.width, self.height),
                                                  edgecolor='k',
                                                  facecolor='w')
                     else:
                         self.figure.clf()
                     n = 0
                     for y in range(self.nrows):
                         for x in range(self.ncols):
                             if n >= self.dataOut.nChannels:
                                 break
                             ax = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan), 1, colspan)
                             if self.showprofile:
                                 ax.ax_profile = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan+colspan), 1, 1)
                             ax.firsttime = True
                             self.axes.append(ax)
                             n += 1
                 def plot(self):
                     if self.xaxis == "frequency":
                         x = self.dataOut.getFreqRange(1)/1000.
                         xlabel = "Frequency (kHz)"
                     elif self.xaxis == "time":
                         x = self.dataOut.getAcfRange(1)
                         xlabel = "Time (ms)"
                     else:
                         x = self.dataOut.getVelRange(1)
                         xlabel = "Velocity (m/s)"
                     y = self.dataOut.getHeiRange()
                     z = self.data[self.CODE]
                     for n, ax in enumerate(self.axes):
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else np.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.ymin = self.ymin if self.ymin else np.nanmin(y)
                             self.ymax = self.ymax if self.ymax else np.nanmax(y)
                             self.zmin = self.zmin if self.zmin else np.nanmin(z)
                             self.zmax = self.zmax if self.zmax else np.nanmax(z)
                             ax.plot = ax.pcolormesh(x, y, z[n].T,
                                                     vmin=self.zmin,
                                                     vmax=self.zmax,
                                                     cmap=plt.get_cmap(self.colormap)
                                                    )
                             divider = make_axes_locatable(ax)
                             cax = divider.new_horizontal(size='3%', pad=0.05)
                             self.figure.add_axes(cax)
                             plt.colorbar(ax.plot, cax)
                             ax.set_xlim(self.xmin, self.xmax)
                             ax.set_ylim(self.ymin, self.ymax)
                             ax.set_ylabel(self.ylabel)
                             ax.set_xlabel(xlabel)
                             ax.firsttime = False
                             if self.showprofile:
                                 ax.plot_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
                                 ax.ax_profile.set_xlim(self.zmin, self.zmax)
                                 ax.ax_profile.set_ylim(self.ymin, self.ymax)
                                 ax.ax_profile.set_xlabel('dB')
                                 ax.ax_profile.grid(b=True, axis='x')
                                 ax.plot_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
                                                                    color="k", linestyle="dashed", lw=2)[0]
                                 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
                         else:
                             ax.plot.set_array(z[n].T.ravel())
                             if self.showprofile:
                                 ax.plot_profile.set_data(self.data['rti'][self.max_time][n], y)
                                 ax.plot_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
                         ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
                                      size=8)
                         self.saveTime = self.max_time
             class PlotCrossSpectraData(PlotData):
                 CODE = 'cspc'
                 zmin_coh = None
                 zmax_coh = None
                 zmin_phase = None
                 zmax_phase = None
                 CONFLATE = False
                 def setup(self):
                     ncolspan = 1
                     colspan = 1
                     self.ncols = 2
                     self.nrows = self.dataOut.nPairs
                     self.width = 3.6*self.ncols
                     self.height = 3.2*self.nrows
                     self.ylabel = 'Range [Km]'
                     self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
                     if self.figure is None:
                         self.figure = plt.figure(figsize=(self.width, self.height),
                                                  edgecolor='k',
                                                  facecolor='w')
                     else:
                         self.figure.clf()
                     for y in range(self.nrows):
                         for x in range(self.ncols):
                             ax = plt.subplot2grid((self.nrows, self.ncols), (y, x), 1, 1)
                             ax.firsttime = True
                             self.axes.append(ax)
                 def plot(self):
                     if self.xaxis == "frequency":
                         x = self.dataOut.getFreqRange(1)/1000.
                         xlabel = "Frequency (kHz)"
                     elif self.xaxis == "time":
                         x = self.dataOut.getAcfRange(1)
                         xlabel = "Time (ms)"
                     else:
                         x = self.dataOut.getVelRange(1)
                         xlabel = "Velocity (m/s)"
                     y = self.dataOut.getHeiRange()
                     z_coh = self.data['cspc_coh']
                     z_phase = self.data['cspc_phase']
                     for n in range(self.nrows):
                         ax = self.axes[2*n]
                         ax1 = self.axes[2*n+1]
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else np.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.ymin = self.ymin if self.ymin else np.nanmin(y)
                             self.ymax = self.ymax if self.ymax else np.nanmax(y)
                             self.zmin_coh = self.zmin_coh if self.zmin_coh else 0.0
                             self.zmax_coh = self.zmax_coh if self.zmax_coh else 1.0
                             self.zmin_phase = self.zmin_phase if self.zmin_phase else -180
                             self.zmax_phase = self.zmax_phase if self.zmax_phase else 180
                             ax.plot = ax.pcolormesh(x, y, z_coh[n].T,
                                                     vmin=self.zmin_coh,
                                                     vmax=self.zmax_coh,
                                                     cmap=plt.get_cmap(self.colormap_coh)
                                                    )
                             divider = make_axes_locatable(ax)
                             cax = divider.new_horizontal(size='3%', pad=0.05)
                             self.figure.add_axes(cax)
                             plt.colorbar(ax.plot, cax)
                             ax.set_xlim(self.xmin, self.xmax)
                             ax.set_ylim(self.ymin, self.ymax)
                             ax.set_ylabel(self.ylabel)
                             ax.set_xlabel(xlabel)
                             ax.firsttime = False
                             ax1.plot = ax1.pcolormesh(x, y, z_phase[n].T,
                                                     vmin=self.zmin_phase,
                                                     vmax=self.zmax_phase,
                                                     cmap=plt.get_cmap(self.colormap_phase)
                                                    )
                             divider = make_axes_locatable(ax1)
                             cax = divider.new_horizontal(size='3%', pad=0.05)
                             self.figure.add_axes(cax)
                             plt.colorbar(ax1.plot, cax)
                             ax1.set_xlim(self.xmin, self.xmax)
                             ax1.set_ylim(self.ymin, self.ymax)
                             ax1.set_ylabel(self.ylabel)
                             ax1.set_xlabel(xlabel)
                             ax1.firsttime = False
                         else:
                             ax.plot.set_array(z_coh[n].T.ravel())
                             ax1.plot.set_array(z_phase[n].T.ravel())
                         ax.set_title('Coherence Ch{} * Ch{}'.format(self.dataOut.pairsList[n][0], self.dataOut.pairsList[n][1]), size=8)
                         ax1.set_title('Phase Ch{} * Ch{}'.format(self.dataOut.pairsList[n][0], self.dataOut.pairsList[n][1]), size=8)
                         self.saveTime = self.max_time
             class PlotSpectraMeanData(PlotSpectraData):
                 CODE = 'spc_mean'
                 colormap = 'jet'
                 def plot(self):
                     if self.xaxis == "frequency":
                         x = self.dataOut.getFreqRange(1)/1000.
                         xlabel = "Frequency (kHz)"
                     elif self.xaxis == "time":
                         x = self.dataOut.getAcfRange(1)
                         xlabel = "Time (ms)"
                     else:
                         x = self.dataOut.getVelRange(1)
                         xlabel = "Velocity (m/s)"
                     y = self.dataOut.getHeiRange()
                     z = self.data['spc']
                     mean = self.data['mean'][self.max_time]
                     for n, ax in enumerate(self.axes):
                         if ax.firsttime:
                             self.xmax = self.xmax if self.xmax else np.nanmax(x)
                             self.xmin = self.xmin if self.xmin else -self.xmax
                             self.ymin = self.ymin if self.ymin else np.nanmin(y)
                             self.ymax = self.ymax if self.ymax else np.nanmax(y)
                             self.zmin = self.zmin if self.zmin else np.nanmin(z)
                             self.zmax = self.zmax if self.zmax else np.nanmax(z)
                             ax.plt = ax.pcolormesh(x, y, z[n].T,
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=plt.get_cmap(self.colormap)
                                                    )
                             ax.plt_dop = ax.plot(mean[n], y,
                                                  color='k')[0]
                             divider = make_axes_locatable(ax)
                             cax = divider.new_horizontal(size='3%', pad=0.05)
                             self.figure.add_axes(cax)
                             plt.colorbar(ax.plt, cax)
                             ax.set_xlim(self.xmin, self.xmax)
                             ax.set_ylim(self.ymin, self.ymax)
                             ax.set_ylabel(self.ylabel)
                             ax.set_xlabel(xlabel)
                             ax.firsttime = False
                             if self.showprofile:
                                 ax.plt_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
                                 ax.ax_profile.set_xlim(self.zmin, self.zmax)
                                 ax.ax_profile.set_ylim(self.ymin, self.ymax)
                                 ax.ax_profile.set_xlabel('dB')
                                 ax.ax_profile.grid(b=True, axis='x')
                                 ax.plt_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
                                                                    color="k", linestyle="dashed", lw=2)[0]
                                 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
                         else:
                             ax.plt.set_array(z[n].T.ravel())
                             ax.plt_dop.set_data(mean[n], y)
                             if self.showprofile:
                                 ax.plt_profile.set_data(self.data['rti'][self.max_time][n], y)
                                 ax.plt_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
                         ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
                                      size=8)
                         self.saveTime = self.max_time
             class PlotRTIData(PlotData):
                 CODE = 'rti'
                 colormap = 'jro'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = self.dataOut.nChannels
                     self.width = 10
                     self.height = 2.2*self.nrows if self.nrows<6 else 12
                     if self.nrows==1:
                         self.height += 1
                     self.ylabel = 'Range [Km]'
                     self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
                     if self.figure is None:
                         self.figure = plt.figure(figsize=(self.width, self.height),
                                                  edgecolor='k',
                                                  facecolor='w')
                     else:
                         self.figure.clf()
                         self.axes = []
                     for n in range(self.nrows):
                         ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
                         ax.firsttime = True
                         self.axes.append(ax)
                 def plot(self):
                     self.x = np.array(self.times)
                     self.y = self.dataOut.getHeiRange()
                     self.z = []
                     for ch in range(self.nrows):
                         self.z.append([self.data[self.CODE][t][ch] for t in self.times])
                     self.z = np.array(self.z)
                     for n, ax in enumerate(self.axes):
                         x, y, z = self.fill_gaps(*self.decimate())
                         xmin = self.min_time
                         xmax = xmin+self.xrange*60*60
+                        self.zmin = self.zmin if self.zmin else np.min(self.z)
+                        self.zmax = self.zmax if self.zmax else np.max(self.z)
                         if ax.firsttime:
                             self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
                             self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
-                            self.zmin = self.zmin if self.zmin else np.nanmin(self.z)
-                            self.zmax = self.zmax if self.zmax else np.nanmax(self.z)
                             plot = ax.pcolormesh(x, y, z[n].T,
                                                  vmin=self.zmin,
                                                  vmax=self.zmax,
                                                  cmap=plt.get_cmap(self.colormap)
                                                 )
                             divider = make_axes_locatable(ax)
                             cax = divider.new_horizontal(size='2%', pad=0.05)
                             self.figure.add_axes(cax)
                             plt.colorbar(plot, cax)
                             ax.set_ylim(self.ymin, self.ymax)
                             ax.xaxis.set_major_formatter(FuncFormatter(func))
                             ax.xaxis.set_major_locator(LinearLocator(6))
                             ax.set_ylabel(self.ylabel)
                             # if self.xmin is None:
                             #     xmin = self.min_time
                             # else:
                             #     xmin = (datetime.datetime.combine(self.dataOut.datatime.date(),
                             #                                      datetime.time(self.xmin, 0, 0))-d1970).total_seconds()
                             ax.set_xlim(xmin, xmax)
                             ax.firsttime = False
                         else:
                             ax.collections.remove(ax.collections[0])
                             ax.set_xlim(xmin, xmax)
                             plot = ax.pcolormesh(x, y, z[n].T,
                                                  vmin=self.zmin,
                                                  vmax=self.zmax,
                                                  cmap=plt.get_cmap(self.colormap)
                                                 )
                         ax.set_title('{} {}'.format(self.titles[n],
                                                     datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
                                      size=8)
                         self.saveTime = self.min_time
             class PlotCOHData(PlotRTIData):
                 CODE = 'coh'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = self.dataOut.nPairs
                     self.width = 10
                     self.height = 2.2*self.nrows if self.nrows<6 else 12
                     if self.nrows==1:
                         self.height += 1
                     self.ylabel = 'Range [Km]'
                     self.titles = ['{} Ch{} * Ch{}'.format(self.CODE.upper(), x[0], x[1]) for x in self.dataOut.pairsList]
                     if self.figure is None:
                         self.figure = plt.figure(figsize=(self.width, self.height),
                                                  edgecolor='k',
                                                  facecolor='w')
                     else:
                         self.figure.clf()
                         self.axes = []
                     for n in range(self.nrows):
                         ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
                         ax.firsttime = True
                         self.axes.append(ax)
             class PlotNoiseData(PlotData):
                 CODE = 'noise'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = 1
                     self.width = 10
                     self.height = 3.2
                     self.ylabel = 'Intensity [dB]'
                     self.titles = ['Noise']
                     if self.figure is None:
                         self.figure = plt.figure(figsize=(self.width, self.height),
                                                  edgecolor='k',
                                                  facecolor='w')
                     else:
                         self.figure.clf()
                         self.axes = []
                     self.ax = self.figure.add_subplot(self.nrows, self.ncols, 1)
                     self.ax.firsttime = True
                 def plot(self):
                     x = self.times
                     xmin = self.min_time
                     xmax = xmin+self.xrange*60*60
                     if self.ax.firsttime:
                         for ch in self.dataOut.channelList:
                             y = [self.data[self.CODE][t][ch] for t in self.times]
                             self.ax.plot(x, y, lw=1, label='Ch{}'.format(ch))
                         self.ax.firsttime = False
                         self.ax.xaxis.set_major_formatter(FuncFormatter(func))
                         self.ax.xaxis.set_major_locator(LinearLocator(6))
                         self.ax.set_ylabel(self.ylabel)
                         plt.legend()
                     else:
                         for ch in self.dataOut.channelList:
                             y = [self.data[self.CODE][t][ch] for t in self.times]
                             self.ax.lines[ch].set_data(x, y)
                     self.ax.set_xlim(xmin, xmax)
                     self.ax.set_ylim(min(y)-5, max(y)+5)
                     self.saveTime = self.min_time
             class PlotWindProfilerData(PlotRTIData):
                 CODE = 'wind'
                 colormap = 'seismic'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = self.dataOut.data_output.shape[0]
                     self.width = 10
                     self.height = 2.2*self.nrows
                     self.ylabel = 'Height [Km]'
                     self.titles = ['Zonal' ,'Meridional', 'Vertical']
                     self.clabels = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
                     self.windFactor = [1, 1, 100]
                     if self.figure is None:
                         self.figure = plt.figure(figsize=(self.width, self.height),
                                                  edgecolor='k',
                                                  facecolor='w')
                     else:
                         self.figure.clf()
                         self.axes = []
                     for n in range(self.nrows):
                         ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
                         ax.firsttime = True
                         self.axes.append(ax)
                 def plot(self):
                     self.x = np.array(self.times)
                     self.y = self.dataOut.heightList
                     self.z = []
                     for ch in range(self.nrows):
                         self.z.append([self.data[self.CODE][t][ch] for t in self.times])
                     self.z = np.array(self.z)
                     self.z = numpy.ma.masked_invalid(self.z)
                     cmap=plt.get_cmap(self.colormap)
                     cmap.set_bad('white', 1.)
                     for n, ax in enumerate(self.axes):
                         x, y, z = self.fill_gaps(*self.decimate())
                         xmin = self.min_time
                         xmax = xmin+self.xrange*60*60
                         if ax.firsttime:
                             self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
                             self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
                             self.zmax = self.zmax if self.zmax else numpy.nanmax(abs(self.z[:-1, :]))
                             self.zmin = self.zmin if self.zmin else -self.zmax
                             plot = ax.pcolormesh(x, y, z[n].T*self.windFactor[n],
                                                  vmin=self.zmin,
                                                  vmax=self.zmax,
                                                  cmap=cmap
                                                 )
                             divider = make_axes_locatable(ax)
                             cax = divider.new_horizontal(size='2%', pad=0.05)
                             cax.set_ylabel(self.clabels[n])
                             self.figure.add_axes(cax)
                             plt.colorbar(plot, cax)
                             ax.set_ylim(self.ymin, self.ymax)
                             ax.xaxis.set_major_formatter(FuncFormatter(func))
                             ax.xaxis.set_major_locator(LinearLocator(6))
                             ax.set_ylabel(self.ylabel)
                             ax.set_xlim(xmin, xmax)
                             ax.firsttime = False
                         else:
                             ax.collections.remove(ax.collections[0])
                             ax.set_xlim(xmin, xmax)
                             plot = ax.pcolormesh(x, y, z[n].T*self.windFactor[n],
                                                  vmin=self.zmin,
                                                  vmax=self.zmax,
                                                  cmap=plt.get_cmap(self.colormap)
                                                 )
                         ax.set_title('{} {}'.format(self.titles[n],
                                                     datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
                                      size=8)
                         self.saveTime = self.min_time
             class PlotSNRData(PlotRTIData):
                 CODE = 'snr'
                 colormap = 'jet'
             class PlotDOPData(PlotRTIData):
                 CODE = 'dop'
                 colormap = 'jet'
             class PlotPHASEData(PlotCOHData):
                 CODE = 'phase'
                 colormap = 'seismic'

schainpy/model/io/jroIO_base.py +12 -5

             '''
             Created on Jul 2, 2014
             @author: roj-idl71
             '''
             import os
             import sys
             import glob
             import time
             import numpy
             import fnmatch
             import time, datetime
             #import h5py
             import traceback
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
             LOCALTIME = True
             def isNumber(cad):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float( cad )
                     return True
                 except:
                     return False
             def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
                 """
                 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                     endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 try:
                     fp = open(filename,'rb')
                 except IOError:
                     print "The file %s can't be opened" %(filename)
                     return 0
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 if not(sts):
                     print "Skipping the file %s because it has not a valid header" %(filename)
                     return 0
                 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                     return 0
                 return 1
             def isTimeInRange(thisTime, startTime, endTime):
                 if endTime >= startTime:
                     if (thisTime < startTime) or (thisTime > endTime):
                         return 0
                     return 1
                 else:
                     if (thisTime < startTime) and (thisTime > endTime):
                         return 0
                     return 1
             def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                     startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                     endTime            :    tiempo final del rango seleccionado en formato datetime.time
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 try:
                     fp = open(filename,'rb')
                 except IOError:
                     print "The file %s can't be opened" %(filename)
                     return None
                 firstBasicHeaderObj = BasicHeader(LOCALTIME)
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
                 processingHeaderObj = ProcessingHeader()
                 lastBasicHeaderObj = BasicHeader(LOCALTIME)
                 sts = firstBasicHeaderObj.read(fp)
                 if not(sts):
                     print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
                     return None
                 if not systemHeaderObj.read(fp):
                     return None
                 if not radarControllerHeaderObj.read(fp):
                     return None
                 if not processingHeaderObj.read(fp):
                     return None
                 filesize = os.path.getsize(filename)
                 offset = processingHeaderObj.blockSize + 24 #header size
                 if filesize <= offset:
                     print "[Reading] %s: This file has not enough data" %filename
                     return None
                 fp.seek(-offset, 2)
                 sts = lastBasicHeaderObj.read(fp)
                 fp.close()
                 thisDatetime = lastBasicHeaderObj.datatime
                 thisTime_last_block = thisDatetime.time()
                 thisDatetime = firstBasicHeaderObj.datatime
                 thisDate = thisDatetime.date()
                 thisTime_first_block = thisDatetime.time()
                 #General case
                 #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                 #-----------o----------------------------o-----------
                 #       startTime                     endTime
                 if endTime >= startTime:
                     if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
                         return None
                     return thisDatetime
                 #If endTime < startTime then endTime belongs to the next day
                 #<<<<<<<<<<<o                            o>>>>>>>>>>>
                 #-----------o----------------------------o-----------
                 #        endTime                    startTime
                 if (thisDate == startDate) and (thisTime_last_block < startTime):
                     return None
                 if (thisDate == endDate) and (thisTime_first_block > endTime):
                     return None
                 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
                     return None
                 return thisDatetime
             def isFolderInDateRange(folder, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     folder            :    nombre completo del directorio.
                                            Su formato deberia ser "/path_root/?YYYYDDD"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el directorio no tiene el formato adecuado
                 """
                 basename = os.path.basename(folder)
                 if not isRadarFolder(basename):
                     print "The folder %s has not the rigth format" %folder
                     return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFolder(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def isFileInDateRange(filename, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                                            Su formato deberia ser "?YYYYDDDsss"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                                                 sss     :    set
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no tiene el formato adecuado
                 """
                 basename = os.path.basename(filename)
                 if not isRadarFile(basename):
                     print "The filename %s has not the rigth format" %filename
                     return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFile(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def getFileFromSet(path, ext, set):
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     try:
                         year = int(thisFile[1:5])
                         doy  = int(thisFile[5:8])
                     except:
                         continue
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
                 if len(myfile)!= 0:
                     return myfile[0]
                 else:
                     filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
                     print 'the filename %s does not exist'%filename
                     print '...going to the last file: '
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def getlastFileFromPath(path, ext):
                 """
                 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     year = thisFile[1:5]
                     if not isNumber(year):
                         continue
                     doy = thisFile[5:8]
                     if not isNumber(doy):
                         continue
                     year = int(year)
                     doy = int(doy)
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def checkForRealPath(path, foldercounter, year, doy, set, ext):
                 """
                 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
                 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
                 el path exacto de un determinado file.
                 Example    :
                     nombre correcto del file es  .../.../D2009307/P2009307367.ext
                     Entonces la funcion prueba con las siguientes combinaciones
                         .../.../y2009307367.ext
                         .../.../Y2009307367.ext
                         .../.../x2009307/y2009307367.ext
                         .../.../x2009307/Y2009307367.ext
                         .../.../X2009307/y2009307367.ext
                         .../.../X2009307/Y2009307367.ext
                     siendo para este caso, la ultima combinacion de letras, identica al file buscado
                 Return:
                     Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
                     caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
                     para el filename
                 """
                 fullfilename = None
                 find_flag = False
                 filename = None
                 prefixDirList = [None,'d','D']
                 if ext.lower() == ".r": #voltage
                     prefixFileList = ['d','D']
                 elif ext.lower() == ".pdata": #spectra
                     prefixFileList = ['p','P']
                 else:
                     return None, filename
                 #barrido por las combinaciones posibles
                 for prefixDir in prefixDirList:
                     thispath = path
                     if prefixDir != None:
                         #formo el nombre del directorio xYYYYDDD (x=d o x=D)
                         if foldercounter == 0:
                             thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
                         else:
                             thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
                     for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
                         filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
                         fullfilename = os.path.join( thispath, filename ) #formo el path completo
                         if os.path.exists( fullfilename ): #verifico que exista
                             find_flag = True
                             break
                     if find_flag:
                         break
                 if not(find_flag):
                     return None, filename
                 return fullfilename, filename
             def isRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return 0
                 return 1
             def isRadarFile(file):
                     try:
                         year = int(file[1:5])
                         doy = int(file[5:8])
                         set = int(file[8:11])
                     except:
                         return 0
                     return 1
             def getDateFromRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
                 return thisDate
             def getDateFromRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
                 return thisDate
             class JRODataIO:
                 c = 3E8
                 isConfig = False
                 basicHeaderObj = None
                 systemHeaderObj = None
                 radarControllerHeaderObj = None
                 processingHeaderObj = None
                 dtype = None
                 pathList = []
                 filenameList = []
                 filename = None
                 ext = None
                 flagIsNewFile = 1
                 flagDiscontinuousBlock = 0
                 flagIsNewBlock = 0
                 fp = None
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 versionFile = 1103
                 fileSize = None
             #     ippSeconds = None
                 fileSizeByHeader = None
                 fileIndex = None
                 profileIndex = None
                 blockIndex = None
                 nTotalBlocks = None
                 maxTimeStep = 30
                 lastUTTime = None
                 datablock = None
                 dataOut = None
                 blocksize = None
                 getByBlock = False
                 def __init__(self):
                     raise NotImplementedError
                 def run(self):
                     raise NotImplementedError
                 def getDtypeWidth(self):
                     dtype_index = get_dtype_index(self.dtype)
                     dtype_width = get_dtype_width(dtype_index)
                     return dtype_width
             class JRODataReader(JRODataIO):
                 online = 0
                 realtime = 0
                 nReadBlocks = 0
                 delay  = 10   #number of seconds waiting a new file
                 nTries  = 3  #quantity tries
                 nFiles = 3   #number of files for searching
                 path = None
                 foldercounter = 0
                 flagNoMoreFiles = 0
                 datetimeList = []
                 __isFirstTimeOnline = 1
                 __printInfo = True
                 profileIndex = None
                 nTxs = 1
                 txIndex = None
                 #Added--------------------
                 selBlocksize = None
                 selBlocktime = None
                 def __init__(self):
                     """
                     This class is used to find data files
                     Example:
                         reader = JRODataReader()
                         fileList = reader.findDataFiles()
                     """
                     pass
                 def createObjByDefault(self):
                     """
                     """
                     raise NotImplementedError
                 def getBlockDimension(self):
                     raise NotImplementedError
                 def __searchFilesOffLine(self,
                                         path,
                                         startDate=None,
                                         endDate=None,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.r',
                                         queue=None,
                                         cursor=None,
                                         skip=None,
                                         walk=True):
                     self.filenameList = []
                     self.datetimeList = []
                     pathList = []
                     dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
                     if dateList == []:
             #             print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
                         return None, None
                     if len(dateList) > 1:
                         print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
                     else:
                         print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
                     filenameList = []
                     datetimeList = []
                     for thisPath in pathList:
             #             thisPath = pathList[pathDict[file]]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         skippedFileList = []
                         if cursor is not None and skip is not None:
                             # if cursor*skip > len(fileList):
                             if skip == 0:
                                 if queue is not None:
                                     queue.put(len(fileList))
                                 skippedFileList = []
                             else:
                                 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
                         else:
                             skippedFileList = fileList
                         for file in skippedFileList:
                             filename = os.path.join(thisPath,file)
                             if not isFileInDateRange(filename, startDate, endDate):
                                 continue
                             thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
                         return None, None
                     print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
                     print
                     for i in range(len(filenameList)):
                         print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
                     """
                     Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
                     devuelve el archivo encontrado ademas de otros datos.
                     Input:
                         path           :    carpeta donde estan contenidos los files que contiene data
                         expLabel        :     Nombre del subexperimento (subfolder)
                         ext              :    extension de los files
                         walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                     Return:
                         directory   :    eL directorio donde esta el file encontrado
                         filename    :    el ultimo file de una determinada carpeta
                         year        :    el anho
                         doy         :    el numero de dia del anho
                         set         :    el set del archivo
                     """
                     if not os.path.isdir(path):
                         return None, None, None, None, None, None
                     dirList = []
                     if not walk:
                         fullpath = path
                         foldercounter = 0
                     else:
                         #Filtra solo los directorios
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isRadarFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None, None, None, None, None
                         dirList = sorted( dirList, key=str.lower )
                         doypath = dirList[-1]
                         foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
                         fullpath = os.path.join(path, doypath, expLabel)
                     print "[Reading] %s folder was found: " %(fullpath )
                     if set == None:
                         filename = getlastFileFromPath(fullpath, ext)
                     else:
                         filename = getFileFromSet(fullpath, ext, set)
                     if not(filename):
                         return None, None, None, None, None, None
                     print "[Reading] %s file was found" %(filename)
                     if not(self.__verifyFile(os.path.join(fullpath, filename))):
                         return None, None, None, None, None, None
                     year = int( filename[1:5] )
                     doy  = int( filename[5:8] )
                     set  = int( filename[8:11] )
                     return fullpath, foldercounter, filename, year, doy, set
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
             #                 print "[Reading] No more Files"
                             return 0
                         filename = self.filenameList[idFile]
                         if not(self.__verifyFile(filename)):
                             continue
                         fileSize = os.path.getsize(filename)
                         fp = open(filename,'rb')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fp = fp
             #         print "[Reading] Setting the file: %s"%self.filename
                     return 1
                 def __setNextFileOnline(self):
                     """
                     Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
                     no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
                     siguientes.
                     Affected:
                         self.flagIsNewFile
                         self.filename
                         self.fileSize
                         self.fp
                         self.set
                         self.flagNoMoreFiles
                     Return:
 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
 : si el file fue abierto con exito y esta listo a ser leido
                     Excepciones:
                         Si un determinado file no puede ser abierto
                     """
                     nFiles = 0
                     fileOk_flag = False
                     firstTime_flag = True
                     self.set += 1
                     if self.set > 999:
                         self.set = 0
                         self.foldercounter += 1
                     #busca el 1er file disponible
                     fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                     if fullfilename:
                         if self.__verifyFile(fullfilename, False):
                             fileOk_flag = True
                     #si no encuentra un file entonces espera y vuelve a buscar
                     if not(fileOk_flag):
                         for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
                             if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
                                 tries = self.nTries
                             else:
                                 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
                             for nTries in range( tries ):
                                 if firstTime_flag:
                                     print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
                                     sleep( self.delay )
                                 else:
                                     print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
                                 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
                                 if fullfilename:
                                     if self.__verifyFile(fullfilename):
                                         fileOk_flag = True
                                         break
                             if fileOk_flag:
                                 break
                             firstTime_flag = False
                             print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
                             self.set += 1
                             if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
                                 self.set = 0
                                 self.doy += 1
                                 self.foldercounter = 0
                     if fileOk_flag:
                         self.fileSize = os.path.getsize( fullfilename )
                         self.filename = fullfilename
                         self.flagIsNewFile = 1
                         if self.fp != None: self.fp.close()
                         self.fp = open(fullfilename, 'rb')
                         self.flagNoMoreFiles = 0
             #             print '[Reading] Setting the file: %s' % fullfilename
                     else:
                         self.fileSize = 0
                         self.filename = None
                         self.flagIsNewFile = 0
                         self.fp = None
                         self.flagNoMoreFiles = 1
             #             print '[Reading] No more files to read'
                     return fileOk_flag
                 def setNextFile(self):
                     if self.fp != None:
                         self.fp.close()
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         print '[Reading] No more files to read'
                         return 0
-                    print '[Reading] Setting the file: %s' % self.filename
+                    if self.verbose:
+                        print '[Reading] Setting the file: %s' % self.filename
                     self.__readFirstHeader()
                     self.nReadBlocks = 0
                     return 1
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.basicHeaderObj.read(self.fp)
                             return 1
                         if self.fileSize == self.fileSizeByHeader:
             #                self.flagEoF = True
                             return 0
                         print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         sleep( self.delay )
                     return 0
                 def waitDataBlock(self,pointer_location):
                     currentPointer = pointer_location
                     neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             return 1
                         print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
                         sleep( self.delay )
                     return 0
                 def __jumpToLastBlock(self):
                     if not(self.__isFirstTimeOnline):
                         return
                     csize = self.fileSize - self.fp.tell()
                     blocksize = self.processingHeaderObj.blockSize
                     #salta el primer bloque de datos
                     if csize > self.processingHeaderObj.blockSize:
                         self.fp.seek(self.fp.tell() + blocksize)
                     else:
                         return
                     csize = self.fileSize - self.fp.tell()
                     neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     while True:
                         if self.fp.tell()<self.fileSize:
                             self.fp.seek(self.fp.tell() + neededsize)
                         else:
                             self.fp.seek(self.fp.tell() - neededsize)
                             break
             #        csize = self.fileSize - self.fp.tell()
             #        neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
             #        factor = int(csize/neededsize)
             #        if factor > 0:
             #            self.fp.seek(self.fp.tell() + factor*neededsize)
                     self.flagIsNewFile = 0
                     self.__isFirstTimeOnline = 0
                 def __setNewBlock(self):
                     if self.fp == None:
                         return 0
             #         if self.online:
             #             self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if self.realtime:
                         self.flagDiscontinuousBlock = 1
                         if not(self.setNextFile()):
                             return 0
                         else:
                             return 1
                     currentSize = self.fileSize - self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     if (currentSize >= neededSize):
                         self.basicHeaderObj.read(self.fp)
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if self.__waitNewBlock():
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
                     self.lastUTTime = self.basicHeaderObj.utc
                     self.flagDiscontinuousBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagDiscontinuousBlock = 1
                     return 1
                 def readNextBlock(self):
                     #Skip block out of startTime and endTime
                     while True:
                         if not(self.__setNewBlock()):
                             return 0
                         if not(self.readBlock()):
                             return 0
                         self.getBasicHeader()
                         if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
                             print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
                                                                           self.processingHeaderObj.dataBlocksPerFile,
                                                                           self.dataOut.datatime.ctime())
                             continue
                         break
-                    print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
+                    if self.verbose:
-                                                              self.processingHeaderObj.dataBlocksPerFile,
+                        print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
-                                                              self.dataOut.datatime.ctime())
+                                                                  self.processingHeaderObj.dataBlocksPerFile,
+                                                                  self.dataOut.datatime.ctime())
                     return 1
                 def __readFirstHeader(self):
                     self.basicHeaderObj.read(self.fp)
                     self.systemHeaderObj.read(self.fp)
                     self.radarControllerHeaderObj.read(self.fp)
                     self.processingHeaderObj.read(self.fp)
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
                     else:
                         raise ValueError, 'Data type was not defined'
                     self.dtype = datatype_str
                     #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
             #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
             #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def __verifyFile(self, filename, msgFlag=True):
                     msg = None
                     try:
                         fp = open(filename, 'rb')
                     except IOError:
                         if msgFlag:
                             print "[Reading] File %s can't be opened" % (filename)
                         return False
                     currentPosition = fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                     if neededSize == 0:
                         basicHeaderObj = BasicHeader(LOCALTIME)
                         systemHeaderObj = SystemHeader()
                         radarControllerHeaderObj = RadarControllerHeader()
                         processingHeaderObj = ProcessingHeader()
                         if not( basicHeaderObj.read(fp) ):
                             fp.close()
                             return False
                         if not( systemHeaderObj.read(fp) ):
                             fp.close()
                             return False
                         if not( radarControllerHeaderObj.read(fp) ):
                             fp.close()
                             return False
                         if not( processingHeaderObj.read(fp) ):
                             fp.close()
                             return False
                         neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
                     else:
                         msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
                     fp.close()
                     fileSize = os.path.getsize(filename)
                     currentSize = fileSize - currentPosition
                     if currentSize < neededSize:
                         if msgFlag and (msg != None):
                             print msg
                         return False
                     return True
                 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
                     path_empty = True
                     dateList = []
                     pathList = []
                     multi_path = path.split(',')
                     if not walk:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             fileList = glob.glob1(single_path, "*"+ext)
                             if not fileList:
                                 continue
                             path_empty = False
                             fileList.sort()
                             for thisFile in fileList:
                                 if not os.path.isfile(os.path.join(single_path, thisFile)):
                                     continue
                                 if not isRadarFile(thisFile):
                                     continue
                                 if not isFileInDateRange(thisFile, startDate, endDate):
                                     continue
                                 thisDate = getDateFromRadarFile(thisFile)
                                 if thisDate in dateList:
                                     continue
                                 dateList.append(thisDate)
                                 pathList.append(single_path)
                     else:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             dirList = []
                             for thisPath in os.listdir(single_path):
                                 if not os.path.isdir(os.path.join(single_path,thisPath)):
                                     continue
                                 if not isRadarFolder(thisPath):
                                     continue
                                 if not isFolderInDateRange(thisPath, startDate, endDate):
                                     continue
                                 dirList.append(thisPath)
                             if not dirList:
                                 continue
                             dirList.sort()
                             for thisDir in dirList:
                                 datapath = os.path.join(single_path, thisDir, expLabel)
                                 fileList = glob.glob1(datapath, "*"+ext)
                                 if not fileList:
                                     continue
                                 path_empty = False
                                 thisDate = getDateFromRadarFolder(thisDir)
                                 pathList.append(datapath)
                                 dateList.append(thisDate)
                     dateList.sort()
                     if walk:
                         pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
                     else:
                         pattern_path = multi_path[0]
                     if path_empty:
                         print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
                     else:
                         if not dateList:
                             print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
                     if include_path:
                         return dateList, pathList
                     return dateList
                 def setup(self,
                             path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=None,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True,
                             getblock = False,
                             nTxs = 1,
                             realtime=False,
                             blocksize=None,
                             blocktime=None,
                             queue=None,
                             skip=None,
-                            cursor=None):
+                            cursor=None,
+                            warnings=True,
+                            verbose=True):
                     if path == None:
                         raise ValueError, "[Reading] The path is not valid"
                     if ext == None:
                         ext = self.ext
                     if online:
                         print "[Reading] Searching files in online mode..."
                         for nTries in range( self.nTries ):
                             fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
                             if fullpath:
                                 break
                             print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
                             sleep( self.delay )
                         if not(fullpath):
                             print "[Reading] There 'isn't any valid file in %s" % path
                             return
                         self.year = year
                         self.doy  = doy
                         self.set  = set - 1
                         self.path = path
                         self.foldercounter = foldercounter
                         last_set = None
                     else:
                         print "[Reading] Searching files in offline mode ..."
                         pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk, cursor=cursor,
                                                                            skip=skip, queue=queue)
                         if not(pathList):
             #                 print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
             #                                                         datetime.datetime.combine(startDate,startTime).ctime(),
             #                                                         datetime.datetime.combine(endDate,endTime).ctime())
             #                 sys.exit(-1)
                             self.fileIndex = -1
                             self.pathList = []
                             self.filenameList = []
                             return
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                         file_name = os.path.basename(filenameList[-1])
                         basename, ext = os.path.splitext(file_name)
                         last_set = int(basename[-3:])
                     self.online = online
                     self.realtime = realtime
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     self.getByBlock = getblock
                     self.nTxs = nTxs
                     self.startTime = startTime
                     self.endTime = endTime
                     #Added-----------------
                     self.selBlocksize = blocksize
                     self.selBlocktime = blocktime
+                    # Verbose-----------
+                    self.verbose = verbose
+                    self.warnings = warnings
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
                         elif startDate != None:
                             print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
                         else:
                             print "[Reading] No files"
                             self.fileIndex = -1
                             self.pathList = []
                             self.filenameList = []
                             return
             #         self.getBasicHeader()
                     if last_set != None:
                         self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
                     return
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                     self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
             #         self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
                 def getFirstHeader(self):
                     raise NotImplementedError
                 def getData(self):
                     raise NotImplementedError
                 def hasNotDataInBuffer(self):
                     raise NotImplementedError
                 def readBlock(self):
                     raise NotImplementedError
                 def isEndProcess(self):
                     return self.flagNoMoreFiles
                 def printReadBlocks(self):
                     print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
                 def printTotalBlocks(self):
                     print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
                 def printNumberOfBlock(self):
                     if self.flagIsNewBlock:
                         print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
                                                                   self.processingHeaderObj.dataBlocksPerFile,
                                                                   self.dataOut.datatime.ctime())
                 def printInfo(self):
                     if self.__printInfo == False:
                         return
                     self.basicHeaderObj.printInfo()
                     self.systemHeaderObj.printInfo()
                     self.radarControllerHeaderObj.printInfo()
                     self.processingHeaderObj.printInfo()
                     self.__printInfo = False
                 def run(self, **kwargs):
                     if not(self.isConfig):
             #            self.dataOut = dataOut
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             class JRODataWriter(JRODataIO):
                 """
                 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 blockIndex = 0
                 path = None
                 setFile = None
                 profilesPerBlock = None
                 blocksPerFile = None
                 nWriteBlocks = 0
                 fileDate = None
                 def __init__(self, dataOut=None):
                     raise NotImplementedError
                 def hasAllDataInBuffer(self):
                     raise NotImplementedError
                 def setBlockDimension(self):
                     raise NotImplementedError
                 def writeBlock(self):
                     raise NotImplementedError
                 def putData(self):
                     raise NotImplementedError
                 def getProcessFlags(self):
                     processFlags = 0
                     dtype_index = get_dtype_index(self.dtype)
                     procflag_dtype = get_procflag_dtype(dtype_index)
                     processFlags += procflag_dtype
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code is not None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nCohInt > 1:
                         processFlags += PROCFLAG.COHERENT_INTEGRATION
                     if self.dataOut.type == "Spectra":
                         if self.dataOut.nIncohInt > 1:
                             processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                         if self.dataOut.data_dc is not None:
                             processFlags += PROCFLAG.SAVE_CHANNELS_DC
                         if self.dataOut.flagShiftFFT:
                             processFlags += PROCFLAG.SHIFT_FFT_DATA
                     return processFlags
                 def setBasicHeader(self):
                     self.basicHeaderObj.size = self.basicHeaderSize #bytes
                     self.basicHeaderObj.version = self.versionFile
                     self.basicHeaderObj.dataBlock = self.nTotalBlocks
                     utc = numpy.floor(self.dataOut.utctime)
                     milisecond  = (self.dataOut.utctime - utc)* 1000.0
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = milisecond
                     self.basicHeaderObj.timeZone = self.dataOut.timeZone
                     self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                     self.basicHeaderObj.errorCount = self.dataOut.errorCount
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj self.
                     Return:
                         None
                     """
                     raise NotImplementedError
                 def __writeFirstHeader(self):
                     """
                     Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                     Affected:
                         __dataType
                     Return:
                         None
                     """
             #        CALCULAR PARAMETROS
                     sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                     self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                     self.basicHeaderObj.write(self.fp)
                     self.systemHeaderObj.write(self.fp)
                     self.radarControllerHeaderObj.write(self.fp)
                     self.processingHeaderObj.write(self.fp)
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                     """
                     if self.fp == None:
                         self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                         self.basicHeaderObj.write(self.fp)
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     """
                     Selecciona el bloque siguiente de datos y los escribe en un file
                     Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                     """
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     print "[Writing] Block No. %d/%d" %(self.blockIndex,
                                                         self.processingHeaderObj.dataBlocksPerFile)
                     return 1
                 def setNextFile(self):
                     """
                     Determina el siguiente file que sera escrito
                     Affected:
                         self.filename
                         self.subfolder
                         self.fp
                         self.setFile
                         self.flagIsNewFile
                     Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                     """
                     ext = self.ext
                     path = self.path
                     if self.fp != None:
                         self.fp.close()
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     setFile = self.setFile
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     setFile += 1
                     #If this is a new day it resets some values
                     if self.dataOut.datatime.date() > self.fileDate:
                         setFile = 0
                         self.nTotalBlocks = 0
                     filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
                     filename = os.path.join( path, subfolder, filen )
                     fp = open( filename,'wb' )
                     self.blockIndex = 0
                     #guardando atributos
                     self.filename = filename
                     self.subfolder = subfolder
                     self.fp = fp
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     self.fileDate = self.dataOut.datatime.date()
                     self.setFirstHeader()
                     print '[Writing] Opening file: %s'%self.filename
                     self.__writeFirstHeader()
                     return 1
                 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
                     """
                     Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                     Inputs:
                         path                :    directory where data will be saved
                         profilesPerBlock    :    number of profiles per block
                         set                 :    initial file set
                         datatype            :    An integer number that defines data type:
 : int8  (1 byte)
 : int16 (2 bytes)
 : int32 (4 bytes)
 : int64 (8 bytes)
 : float32 (4 bytes)
 : double64 (8 bytes)
                     Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                     """
                     if ext == None:
                         ext = self.ext
                     self.ext = ext.lower()
                     self.path = path
                     if set is None:
                         self.setFile = -1
                     else:
                         self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile
                     self.profilesPerBlock = profilesPerBlock
                     self.dataOut = dataOut
                     self.fileDate = self.dataOut.datatime.date()
                     #By default
                     self.dtype = self.dataOut.dtype
                     if datatype is not None:
                         self.dtype = get_numpy_dtype(datatype)
                     if not(self.setNextFile()):
                         print "[Writing] There isn't a next file"
                         return 0
                     self.setBlockDimension()
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()

schainpy/model/proc/jroproc_base.py +1 -1

             '''
             $Author: murco $
             $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
             '''
             import inspect
             from fuzzywuzzy import process
             def checkKwargs(method, kwargs):
                 currentKwargs = kwargs
                 choices = inspect.getargspec(method).args
                 try:
                     choices.remove('self')
                 except Exception as e:
                     pass
                 try:
                     choices.remove('dataOut')
                 except Exception as e:
                     pass
                 for kwarg in kwargs:
                     fuzz = process.extractOne(kwarg, choices)
                     if fuzz is None:
                         continue
                     if fuzz[1] < 100:
-                        raise Exception('\x1b[2;30;43mDid you mean {} instead of {} in {}? \x1b[0m'.
+                        raise Exception('\x1b[0;32;40mDid you mean {} instead of {} in {}? \x1b[0m'.
                                         format(fuzz[0], kwarg, method.__self__.__class__.__name__))
             class ProcessingUnit(object):
                 """
                 Esta es la clase base para el procesamiento de datos.
                 Contiene el metodo "call" para llamar operaciones. Las operaciones pueden ser:
                     - Metodos internos (callMethod)
                     - Objetos del tipo Operation (callObject). Antes de ser llamados, estos objetos
                       tienen que ser agreagados con el metodo "add".
                 """
                 # objeto de datos de entrada (Voltage, Spectra o Correlation)
                 dataIn = None
                 dataInList = []
                 # objeto de datos de entrada (Voltage, Spectra o Correlation)
                 dataOut = None
                 operations2RunDict = None
                 isConfig = False
                 def __init__(self, *args, **kwargs):
                     self.dataIn = None
                     self.dataInList = []
                     self.dataOut = None
                     self.operations2RunDict = {}
                     self.operationKwargs = {}
                     self.isConfig = False
                     self.args = args
                     self.kwargs = kwargs
                     checkKwargs(self.run, kwargs)
                 def getAllowedArgs(self):
                     return inspect.getargspec(self.run).args
                 def addOperationKwargs(self, objId, **kwargs):
                     '''
                     '''
                     self.operationKwargs[objId] = kwargs
                 def addOperation(self, opObj, objId):
                     """
                     Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
                     identificador asociado a este objeto.
                     Input:
                         object    :    objeto de la clase "Operation"
                     Return:
                         objId    :    identificador del objeto, necesario para ejecutar la operacion
                     """
                     self.operations2RunDict[objId] = opObj
                     return objId
                 def getOperationObj(self, objId):
                     if objId not in self.operations2RunDict.keys():
                         return None
                     return self.operations2RunDict[objId]
                 def operation(self, **kwargs):
                     """
                     Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
                     atributos del objeto dataOut
                     Input:
                         **kwargs    :    Diccionario de argumentos de la funcion a ejecutar
                     """
                     raise NotImplementedError
                 def callMethod(self, name, opId):
                     """
                     Ejecuta el metodo con el nombre "name" y con argumentos **kwargs de la propia clase.
                     Input:
                         name        :    nombre del metodo a ejecutar
                         **kwargs     :    diccionario con los nombres y valores de la funcion a ejecutar.
                     """
                     #Checking the inputs
                     if name == 'run':
                         if not self.checkInputs():
                             self.dataOut.flagNoData = True
                             return False
                     else:
                         #Si no es un metodo RUN la entrada es la misma dataOut (interna)
                         if self.dataOut is not None and self.dataOut.isEmpty():
                             return False
                     #Getting the pointer to method
                     methodToCall = getattr(self, name)
                     #Executing the self method
                     if hasattr(self, 'mp'):
                         if name=='run':
                             if self.mp is False:
                                 self.mp = True
                                 self.start()
                         else:
                             methodToCall(**self.operationKwargs[opId])
                     else:
                         if name=='run':
                             methodToCall(**self.kwargs)
                         else:
                             methodToCall(**self.operationKwargs[opId])
                     if self.dataOut is None:
                         return False
                     if self.dataOut.isEmpty():
                         return False
                     return True
                 def callObject(self, objId):
                     """
                     Ejecuta la operacion asociada al identificador del objeto "objId"
                     Input:
                         objId        :    identificador del objeto a ejecutar
                         **kwargs    :    diccionario con los nombres y valores de la funcion a ejecutar.
                     Return:
                         None
                     """
                     if self.dataOut is not None and self.dataOut.isEmpty():
                         return False
                     externalProcObj = self.operations2RunDict[objId]
                     if hasattr(externalProcObj, 'mp'):
                         if externalProcObj.mp is False:
                             self.operationKwargs[objId] = externalProcObj.kwargs
                             externalProcObj.mp = True
                             externalProcObj.start()
                     else:
                         externalProcObj.run(self.dataOut, **externalProcObj.kwargs)
                         self.operationKwargs[objId] = externalProcObj.kwargs
                     return True
                 def call(self, opType, opName=None, opId=None):
                     """
                     Return True si ejecuta la operacion interna nombrada "opName" o la operacion externa
                     identificada con el id "opId"; con los argumentos "**kwargs".
                     False si la operacion no se ha ejecutado.
                     Input:
                         opType    :    Puede ser "self" o "external"
                             Depende del tipo de operacion para llamar a:callMethod or callObject:
 . If opType = "self": Llama a un metodo propio de esta clase:
                                     name_method = getattr(self, name)
                                     name_method(**kwargs)
 . If opType = "other" o"external": Llama al metodo "run()" de una instancia de la
                                clase "Operation" o de un derivado de ella:
                                     instanceName = self.operationList[opId]
                                     instanceName.run(**kwargs)
                         opName    : Si la operacion es interna (opType = 'self'), entonces el "opName" sera
                                     usada para llamar a un metodo interno de la clase Processing
                         opId    :    Si la operacion es externa (opType = 'other' o 'external), entonces el
                                     "opId" sera usada para llamar al metodo "run" de la clase Operation
                                     registrada anteriormente con ese Id
                     Exception:
                            Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
                            "addOperation" e identificado con el valor "opId"  = el id de la operacion.
                            De lo contrario retornara un error del tipo ValueError
                     """
                     if opType == 'self':
                         if not opName:
                             raise ValueError, "opName parameter should be defined"
                         sts = self.callMethod(opName, opId)
                     elif opType == 'other' or opType == 'external' or opType == 'plotter':
                         if not opId:
                             raise ValueError, "opId parameter should be defined"
                         if opId not in self.operations2RunDict.keys():
                             raise ValueError, "Any operation with id=%s has been added" %str(opId)
                         sts = self.callObject(opId)
                     else:
                         raise ValueError, "opType should be 'self', 'external' or 'plotter'; and not '%s'" %opType
                     return sts
                 def setInput(self, dataIn):
                     self.dataIn = dataIn
                     self.dataInList.append(dataIn)
                 def getOutputObj(self):
                     return self.dataOut
                 def checkInputs(self):
                     for thisDataIn in self.dataInList:
                         if thisDataIn.isEmpty():
                             return False
                     return True
                 def setup(self):
                     raise NotImplementedError
                 def run(self):
                     raise NotImplementedError
                 def close(self):
                     #Close every thread, queue or any other object here is it is neccesary.
                     return
             class Operation(object):
                 """
                 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
                 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
                 acumulacion dentro de esta clase
                 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
                 """
                 __buffer = None
                 isConfig = False
                 def __init__(self, **kwargs):
                     self.__buffer = None
                     self.isConfig = False
                     self.kwargs = kwargs
                     checkKwargs(self.run, kwargs)
                 def getAllowedArgs(self):
                     return inspect.getargspec(self.run).args
                 def setup(self):
                     self.isConfig = True
                     raise NotImplementedError
                 def run(self, dataIn, **kwargs):
                     """
                     Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
                     atributos del objeto dataIn.
                     Input:
                         dataIn    :    objeto del tipo JROData
                     Return:
                         None
                     Affected:
                         __buffer    :    buffer de recepcion de datos.
                     """
                     if not self.isConfig:
                         self.setup(**kwargs)
                     raise NotImplementedError
                 def close(self):
                     pass

schainpy/model/utils/jroutils_publish.py +20 -8

             '''
             @author: Juan C. Espinoza
             '''
             import time
             import json
             import numpy
             import paho.mqtt.client as mqtt
             import zmq
-            import cPickle as pickle
+            from profilehooks import profile
             import datetime
             from zmq.utils.monitor import recv_monitor_message
             from functools import wraps
             from threading import Thread
             from multiprocessing import Process
             from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
             MAXNUMX = 100
             MAXNUMY = 100
             class PrettyFloat(float):
                 def __repr__(self):
                     return '%.2f' % self
             def roundFloats(obj):
                 if isinstance(obj, list):
                     return map(roundFloats, obj)
                 elif isinstance(obj, float):
                     return round(obj, 2)
-            def decimate(z):
+            def decimate(z, MAXNUMY):
                 # dx = int(len(self.x)/self.__MAXNUMX) + 1
                 dy = int(len(z[0])/MAXNUMY) + 1
                 return z[::, ::dy]
             class throttle(object):
                 """Decorator that prevents a function from being called more than once every
                 time period.
                 To create a function that cannot be called more than once a minute, but
                 will sleep until it can be called:
                 @throttle(minutes=1)
                 def foo():
                   pass
                 for i in range(10):
                   foo()
                   print "This function has run %s times." % i
                 """
                 def __init__(self, seconds=0, minutes=0, hours=0):
                     self.throttle_period = datetime.timedelta(
                         seconds=seconds, minutes=minutes, hours=hours
                     )
                     self.time_of_last_call = datetime.datetime.min
                 def __call__(self, fn):
                     @wraps(fn)
                     def wrapper(*args, **kwargs):
                         now = datetime.datetime.now()
                         time_since_last_call = now - self.time_of_last_call
                         time_left = self.throttle_period - time_since_last_call
                         if time_left > datetime.timedelta(seconds=0):
                             return
                         self.time_of_last_call = datetime.datetime.now()
                         return fn(*args, **kwargs)
                     return wrapper
             class PublishData(Operation):
                 """Clase publish."""
                 def __init__(self, **kwargs):
                     """Inicio."""
                     Operation.__init__(self, **kwargs)
                     self.isConfig = False
                     self.client = None
                     self.zeromq = None
                     self.mqtt = None
                 def on_disconnect(self, client, userdata, rc):
                     if rc != 0:
                         print("Unexpected disconnection.")
                     self.connect()
                 def connect(self):
                     print 'trying to connect'
                     try:
                         self.client.connect(
                             host=self.host,
                             port=self.port,
                             keepalive=60*10,
                             bind_address='')
                         self.client.loop_start()
                         # self.client.publish(
                         #     self.topic + 'SETUP',
                         #     json.dumps(setup),
                         #     retain=True
                         #     )
                     except:
                         print "MQTT Conection error."
                         self.client = False
-                def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, **kwargs):
+                def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, verbose=True, **kwargs):
                     self.counter = 0
                     self.topic = kwargs.get('topic', 'schain')
                     self.delay = kwargs.get('delay', 0)
                     self.plottype = kwargs.get('plottype', 'spectra')
                     self.host = kwargs.get('host', "10.10.10.82")
                     self.port = kwargs.get('port', 3000)
                     self.clientId = clientId
                     self.cnt = 0
                     self.zeromq = zeromq
                     self.mqtt = kwargs.get('plottype', 0)
                     self.client = None
+                    self.verbose = verbose
+                    self.dataOut.firstdata = True
                     setup = []
                     if mqtt is 1:
                         self.client = mqtt.Client(
                             client_id=self.clientId + self.topic + 'SCHAIN',
                             clean_session=True)
                         self.client.on_disconnect = self.on_disconnect
                         self.connect()
                         for plot in self.plottype:
                             setup.append({
                                 'plot': plot,
                                 'topic': self.topic + plot,
                                 'title': getattr(self, plot + '_' + 'title', False),
                                 'xlabel': getattr(self, plot + '_' + 'xlabel', False),
                                 'ylabel': getattr(self, plot + '_' + 'ylabel', False),
                                 'xrange': getattr(self, plot + '_' + 'xrange', False),
                                 'yrange': getattr(self, plot + '_' + 'yrange', False),
                                 'zrange': getattr(self, plot + '_' + 'zrange', False),
                             })
                     if zeromq is 1:
                         context = zmq.Context()
                         self.zmq_socket = context.socket(zmq.PUSH)
                         server = kwargs.get('server', 'zmq.pipe')
                         if 'tcp://' in server:
                             address = server
                         else:
                             address = 'ipc:///tmp/%s' % server
                         self.zmq_socket.connect(address)
                         time.sleep(1)
                 def publish_data(self):
                     self.dataOut.finished = False
                     if self.mqtt is 1:
                         yData = self.dataOut.heightList[:2].tolist()
                         if self.plottype == 'spectra':
                             data = getattr(self.dataOut, 'data_spc')
                             z = data/self.dataOut.normFactor
                             zdB = 10*numpy.log10(z)
                             xlen, ylen = zdB[0].shape
                             dx = int(xlen/MAXNUMX) + 1
                             dy = int(ylen/MAXNUMY) + 1
                             Z = [0 for i in self.dataOut.channelList]
                             for i in self.dataOut.channelList:
                                 Z[i] = zdB[i][::dx, ::dy].tolist()
                             payload = {
                                 'timestamp': self.dataOut.utctime,
                                 'data': roundFloats(Z),
                                 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                 'interval': self.dataOut.getTimeInterval(),
                                 'type': self.plottype,
                                 'yData': yData
                             }
                             # print payload
                         elif self.plottype in ('rti', 'power'):
                             data = getattr(self.dataOut, 'data_spc')
                             z = data/self.dataOut.normFactor
                             avg = numpy.average(z, axis=1)
                             avgdB = 10*numpy.log10(avg)
                             xlen, ylen = z[0].shape
                             dy = numpy.floor(ylen/self.__MAXNUMY) + 1
                             AVG = [0 for i in self.dataOut.channelList]
                             for i in self.dataOut.channelList:
                                 AVG[i] = avgdB[i][::dy].tolist()
                             payload = {
                                 'timestamp': self.dataOut.utctime,
                                 'data': roundFloats(AVG),
                                 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                 'interval': self.dataOut.getTimeInterval(),
                                 'type': self.plottype,
                                 'yData': yData
                             }
                         elif self.plottype == 'noise':
                             noise = self.dataOut.getNoise()/self.dataOut.normFactor
                             noisedB = 10*numpy.log10(noise)
                             payload = {
                                 'timestamp': self.dataOut.utctime,
                                 'data': roundFloats(noisedB.reshape(-1, 1).tolist()),
                                 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                 'interval': self.dataOut.getTimeInterval(),
                                 'type': self.plottype,
                                 'yData': yData
                             }
                         elif self.plottype == 'snr':
                             data = getattr(self.dataOut, 'data_SNR')
                             avgdB = 10*numpy.log10(data)
                             ylen = data[0].size
                             dy = numpy.floor(ylen/self.__MAXNUMY) + 1
                             AVG = [0 for i in self.dataOut.channelList]
                             for i in self.dataOut.channelList:
                                 AVG[i] = avgdB[i][::dy].tolist()
                             payload = {
                                 'timestamp': self.dataOut.utctime,
                                 'data': roundFloats(AVG),
                                 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
                                 'type': self.plottype,
                                 'yData': yData
                             }
                         else:
                             print "Tipo de grafico invalido"
                             payload = {
                                 'data': 'None',
                                 'timestamp': 'None',
                                 'type': None
                             }
                             # print 'Publishing data to {}'.format(self.host)
                         self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
                     if self.zeromq is 1:
-                        print '[Sending] {} - {}'.format(self.dataOut.type, self.dataOut.datatime)
+                        if self.verbose:
+                            print '[Sending] {} - {}'.format(self.dataOut.type, self.dataOut.datatime)
                         self.zmq_socket.send_pyobj(self.dataOut)
+                        self.dataOut.firstdata = False
                 def run(self, dataOut, **kwargs):
                     self.dataOut = dataOut
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.publish_data()
                     time.sleep(self.delay)
                 def close(self):
                     if self.zeromq is 1:
                         self.dataOut.finished = True
                         self.zmq_socket.send_pyobj(self.dataOut)
+                        self.zmq_socket.close()
                     if self.client:
                         self.client.loop_stop()
                         self.client.disconnect()
             class ReceiverData(ProcessingUnit, Process):
                 throttle_value = 5
                 def __init__(self, **kwargs):
                     ProcessingUnit.__init__(self, **kwargs)
                     Process.__init__(self)
                     self.mp = False
                     self.isConfig = False
                     self.isWebConfig = False
                     self.plottypes =[]
                     self.connections = 0
                     server = kwargs.get('server', 'zmq.pipe')
                     plot_server = kwargs.get('plot_server', 'zmq.web')
                     if 'tcp://' in server:
                         address = server
                     else:
                         address = 'ipc:///tmp/%s' % server
                     if 'tcp://' in plot_server:
                         plot_address = plot_server
                     else:
                         plot_address = 'ipc:///tmp/%s' % plot_server
                     self.address = address
                     self.plot_address = plot_address
                     self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
                     self.realtime = kwargs.get('realtime', False)
                     self.throttle_value = kwargs.get('throttle', 5)
                     self.sendData = self.initThrottle(self.throttle_value)
                     self.setup()
                 def setup(self):
                     self.data = {}
                     self.data['times'] = []
                     for plottype in self.plottypes:
                         self.data[plottype] = {}
                     self.data['noise'] = {}
                     self.data['throttle'] = self.throttle_value
                     self.data['ENDED'] = False
                     self.isConfig = True
                     self.data_web = {}
                 def event_monitor(self, monitor):
                     events = {}
                     for name in dir(zmq):
                         if name.startswith('EVENT_'):
                             value = getattr(zmq, name)
                             events[value] = name
                     while monitor.poll():
                         evt = recv_monitor_message(monitor)
                         if evt['event'] == 32:
                             self.connections += 1
                         if evt['event'] == 512:
                             pass
                         if self.connections == 0 and self.started is True:
                             self.ended = True
                         evt.update({'description': events[evt['event']]})
                         if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
                             break
                     monitor.close()
                     print("event monitor thread done!")
                 def initThrottle(self, throttle_value):
                     @throttle(seconds=throttle_value)
                     def sendDataThrottled(fn_sender, data):
                         fn_sender(data)
                     return sendDataThrottled
                 def send(self, data):
                     # print '[sending] data=%s size=%s' % (data.keys(), len(data['times']))
                     self.sender.send_pyobj(data)
-                def update(self):
+                def update(self):
                     t = self.dataOut.utctime
                     if t in self.data['times']:
                         return
                     self.data['times'].append(t)
                     self.data['dataOut'] = self.dataOut
                     for plottype in self.plottypes:
                         if plottype == 'spc':
                             z = self.dataOut.data_spc/self.dataOut.normFactor
                             self.data[plottype] = 10*numpy.log10(z)
                             self.data['noise'][t] = 10*numpy.log10(self.dataOut.getNoise()/self.dataOut.normFactor)
                         if plottype == 'cspc':
                             jcoherence = self.dataOut.data_cspc/numpy.sqrt(self.dataOut.data_spc*self.dataOut.data_spc)
                             self.data['cspc_coh'] = numpy.abs(jcoherence)
                             self.data['cspc_phase'] = numpy.arctan2(jcoherence.imag, jcoherence.real)*180/numpy.pi
                         if plottype == 'rti':
                             self.data[plottype][t] = self.dataOut.getPower()
                         if plottype == 'snr':
                             self.data[plottype][t] = 10*numpy.log10(self.dataOut.data_SNR)
                         if plottype == 'dop':
                             self.data[plottype][t] = 10*numpy.log10(self.dataOut.data_DOP)
                         if plottype == 'mean':
                             self.data[plottype][t] = self.dataOut.data_MEAN
                         if plottype == 'std':
                             self.data[plottype][t] = self.dataOut.data_STD
                         if plottype == 'coh':
                             self.data[plottype][t] = self.dataOut.getCoherence()
                         if plottype == 'phase':
                             self.data[plottype][t] = self.dataOut.getCoherence(phase=True)
                         if plottype == 'wind':
                             self.data[plottype][t] = self.dataOut.data_output
                         if self.realtime:
                             self.data_web['timestamp'] = t
                             if plottype == 'spc':
                                 self.data_web[plottype] = roundFloats(decimate(self.data[plottype]).tolist())
                             elif plottype == 'cspc':
                                 self.data_web['cspc_coh'] = roundFloats(decimate(self.data['cspc_coh']).tolist())
                                 self.data_web['cspc_phase'] = roundFloats(decimate(self.data['cspc_phase']).tolist())
                             elif plottype == 'noise':
                                 self.data_web['noise'] = roundFloats(self.data['noise'][t].tolist())
                             else:
                                 self.data_web[plottype] = roundFloats(decimate(self.data[plottype][t]).tolist())
                             self.data_web['interval'] = self.dataOut.getTimeInterval()
                             self.data_web['type'] = plottype
                 def run(self):
                     print '[Starting] {} from {}'.format(self.name, self.address)
                     self.context = zmq.Context()
                     self.receiver = self.context.socket(zmq.PULL)
                     self.receiver.bind(self.address)
                     monitor = self.receiver.get_monitor_socket()
                     self.sender = self.context.socket(zmq.PUB)
                     if self.realtime:
                         self.sender_web = self.context.socket(zmq.PUB)
                         self.sender_web.connect(self.plot_address)
                         time.sleep(1)
                     self.sender.bind("ipc:///tmp/zmq.plots")
+                    time.sleep(3)
                     t = Thread(target=self.event_monitor, args=(monitor,))
                     t.start()
                     while True:
                         self.dataOut = self.receiver.recv_pyobj()
                         # print '[Receiving] {} - {}'.format(self.dataOut.type,
                         #                                    self.dataOut.datatime.ctime())
                         self.update()
+                        if self.dataOut.firstdata is True:
+                            self.data['STARTED'] = True
                         if self.dataOut.finished is True:
                             self.send(self.data)
                             self.connections -= 1
                             if self.connections == 0 and self.started:
                                 self.ended = True
                                 self.data['ENDED'] = True
                                 self.send(self.data)
                                 self.setup()
+                                self.started = False
                         else:
                             if self.realtime:
                                 self.send(self.data)
                                 self.sender_web.send_string(json.dumps(self.data_web))
                             else:
                                 self.sendData(self.send, self.data)
                             self.started = True
+                        self.data['STARTED'] = False
                     return
                 def sendToWeb(self):
                     if not self.isWebConfig:
                         context = zmq.Context()
                         sender_web_config = context.socket(zmq.PUB)
                         if 'tcp://' in self.plot_address:
                             dum, address, port = self.plot_address.split(':')
                             conf_address = '{}:{}:{}'.format(dum, address, int(port)+1)
                         else:
                             conf_address = self.plot_address + '.config'
                         sender_web_config.bind(conf_address)
                         time.sleep(1)
                         for kwargs in self.operationKwargs.values():
                             if 'plot' in kwargs:
                                 print '[Sending] Config data to web for {}'.format(kwargs['code'].upper())
                                 sender_web_config.send_string(json.dumps(kwargs))
                         self.isWebConfig = True

schainpy/scripts/PPD.py +13 -12

             import argparse
             from schainpy.controller import Project, multiSchain
             desc = "HF_EXAMPLE"
             def fiber(cursor, skip, q, dt):
                 controllerObj = Project()
                 controllerObj.setup(id='191', name='test01', description=desc)
                 readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
-                                                             path='/home/nanosat/data/julia',
+                                                             path='/home/nanosat/data/sp1_f0',
                                                              startDate=dt,
                                                              endDate=dt,
                                                              startTime="00:00:00",
                                                              endTime="23:59:59",
                                                              online=0,
                                                              #set=1426485881,
-                                                             delay=10,
                                                              walk=1,
                                                              queue=q,
                                                              cursor=cursor,
                                                              skip=skip,
+                                                             verbose=1
                                                              #timezone=-5*3600
                                                              )
                 #     #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
                 #
                 procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
                 # procUnitConfObj2.addParameter(name='nipp', value='5', format='int')
                 procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
                 opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
                 #
                 #     opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
                 #     opObj11.addParameter(name='id', value='1000', format='int')
                 #     opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
                 #     opObj11.addParameter(name='channelList', value='0', format='intlist')
                 #     opObj11.addParameter(name='zmin', value='-120', format='float')
                 #     opObj11.addParameter(name='zmax', value='-70', format='float')
                 #     opObj11.addParameter(name='save', value='1', format='int')
                 #     opObj11.addParameter(name='figpath', value=figpath, format='str')
-                opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
+                # opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
-                opObj11.addParameter(name='channelList', value='0', format='intList')
+                # opObj11.addParameter(name='channelList', value='0', format='intList')
-                opObj11.addParameter(name='id', value='2000', format='int')
+                # opObj11.addParameter(name='id', value='2000', format='int')
-                # opObj11.addParameter(name='colormap', value='0', format='bool')
+                # # opObj11.addParameter(name='colormap', value='0', format='bool')
-                opObj11.addParameter(name='onlySNR', value='1', format='bool')
+                # opObj11.addParameter(name='onlySNR', value='1', format='bool')
-                opObj11.addParameter(name='DOP', value='0', format='bool')
+                # opObj11.addParameter(name='DOP', value='0', format='bool')
-                # opObj11.addParameter(name='showSNR', value='1', format='bool')
+                # # opObj11.addParameter(name='showSNR', value='1', format='bool')
-                # opObj11.addParameter(name='SNRthresh', value='0', format='int')
+                # # opObj11.addParameter(name='SNRthresh', value='0', format='int')
                 # opObj11.addParameter(name='SNRmin', value='-10', format='int')
                 # opObj11.addParameter(name='SNRmax', value='30', format='int')
                 # opObj11.addParameter(name='showSNR', value='1', format='int')
                 # # opObj11.addParameter(name='channelList', value='0', format='intlist')
                 # #     opObj11.addParameter(name='xmin', value='0', format='float')
                 # opObj11.addParameter(name='xmin', value='0', format='float')
                 # opObj11.addParameter(name='xmax', value='24', format='float')
                 # opObj11.addParameter(name='zmin', value='-110', format='float')
                 # opObj11.addParameter(name='zmax', value='-70', format='float')
                 # opObj11.addParameter(name='save', value='0', format='int')
                 # # opObj11.addParameter(name='figpath', value='/tmp/', format='str')
                 #
                 opObj12 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
                 opObj12.addParameter(name='zeromq', value=1, format='int')
+                opObj12.addParameter(name='verbose', value=0, format='bool')
                 # opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
                 # opObj13.addParameter(name='zeromq', value=1, format='int')
                 # opObj13.addParameter(name='server', value="juanca", format='str')
-                opObj12.addParameter(name='delay', value=1, format='int')
+                opObj12.addParameter(name='delay', value=0, format='int')
                 # print "Escribiendo el archivo XML"
                 # controllerObj.writeXml(filename)
                 # print "Leyendo el archivo XML"
                 # controllerObj.readXml(filename)
                 # timeit.timeit('controllerObj.run()', number=2)
                 controllerObj.start()
             if __name__ == '__main__':
                 parser = argparse.ArgumentParser(description='Set number of parallel processes')
                 parser.add_argument('--nProcess', default=1, type=int)
                 args = parser.parse_args()
-                multiSchain(fiber, nProcess=args.nProcess, startDate='2015/09/26', endDate='2015/09/26')
+                multiSchain(fiber, nProcess=args.nProcess, startDate='2017/01/26', endDate='2017/01/28')

schainpy/scripts/julia_mp.py +88 -66

@@ -1,75 +1,97
1	#!/usr/bin/env python	1	import argparse
2	'''
3	Created on Jul 7, 2014
4		2
5	@author: roj-idl71	3	from schainpy.controller import Project, multiSchain
6	'''
7	import os, sys
8	from datetime import datetime, timedelta
9	import multiprocessing
10	from schainpy.controller import Project
11		4
12	def main(date):	5	desc = "HF_EXAMPLE"
13		6
14	controllerObj = Project()	7	def fiber(cursor, skip, q, dt):
15
16	controllerObj.setup(id='191', name='test01', description='')
17
18	readUnitConfObj = controllerObj.addReadUnit(datatype='Spectra',
19	path='/home/nanosat/data/zeus',
20	startDate=date,
21	endDate=date,
22	startTime='00:00:00',
23	endTime='23:59:59',
24	online=0,
25	walk=1,
26	expLabel='')
27
28	procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
29	#opObj11 = procUnitConfObj1.addOperation(name='removeDC')
30	#opObj11.addParameter(name='mode', value='1', format='int')
31
32	#opObj11 = procUnitConfObj1.addOperation(name='removeInterference')
33
34
35	opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
36	opObj11.addParameter(name='id', value='10', format='int')
37	opObj11.addParameter(name='wintitle', value='150Km', format='str')
38	opObj11.addParameter(name='colormap', value='jro', format='str')
39	opObj11.addParameter(name='xaxis', value='time', format='str')
40	opObj11.addParameter(name='xmin', value='0', format='int')
41	opObj11.addParameter(name='xmax', value='23', format='int')
42	#opObj11.addParameter(name='ymin', value='100', format='int')
43	#opObj11.addParameter(name='ymax', value='150', format='int')
44	opObj11.addParameter(name='zmin', value='10', format='int')
45	opObj11.addParameter(name='zmax', value='35', format='int')
46		8
		9	controllerObj = Project()
47		10
48		11	controllerObj.setup(id='191', name='test01', description=desc)
49		12
50	opObject12 = procUnitConfObj1.addOperation(name='PlotRTIData', optype='other')	13	readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
51	opObject12.addParameter(name='id', value='12', format='int')	14	path='/home/nanosat/data/julia',
52	opObject12.addParameter(name='wintitle', value='150Km', format='str')	15	startDate=dt,
53	opObject12.addParameter(name='colormap', value='jro', format='str')	16	endDate=dt,
54	opObject12.addParameter(name='xaxis', value='time', format='str')	17	startTime="00:00:00",
55	opObject12.addParameter(name='xmin', value='0', format='int')	18	endTime="23:59:59",
56	opObject12.addParameter(name='xmax', value='23', format='int')	19	online=0,
57	#opObject12.addParameter(name='ymin', value='100', format='int')	20	#set=1426485881,
58	#opObject12.addParameter(name='ymax', value='150', format='int')	21	delay=10,
59	opObject12.addParameter(name='zmin', value='10', format='int')	22	walk=1,
60	opObject12.addParameter(name='zmax', value='35', format='int')	23	queue=q,
61	#opObject12.addParameter(name='pause', value='1', format='bool')	24	cursor=cursor,
62	opObject12.addParameter(name='show', value='0', format='bool')	25	skip=skip,
63	opObject12.addParameter(name='save', value='/tmp', format='str')	26	#timezone=-5*3600
64		27	)
		28
		29	# #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
		30	#
		31	procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
		32	# procUnitConfObj2.addParameter(name='nipp', value='5', format='int')
		33
		34	# procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
		35	# opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
		36
		37	#
		38	# opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
		39	# opObj11.addParameter(name='id', value='1000', format='int')
		40	# opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
		41	# opObj11.addParameter(name='channelList', value='0', format='intlist')
		42	# opObj11.addParameter(name='zmin', value='-120', format='float')
		43	# opObj11.addParameter(name='zmax', value='-70', format='float')
		44	# opObj11.addParameter(name='save', value='1', format='int')
		45	# opObj11.addParameter(name='figpath', value=figpath, format='str')
		46
		47	# opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
		48	# opObj11.addParameter(name='channelList', value='0', format='intList')
		49	#
		50	# opObj11.addParameter(name='id', value='2000', format='int')
		51	# # opObj11.addParameter(name='colormap', value='0', format='bool')
		52	# opObj11.addParameter(name='onlySNR', value='1', format='bool')
		53	# opObj11.addParameter(name='DOP', value='0', format='bool')
		54	# opObj11.addParameter(name='showSNR', value='1', format='bool')
		55	# opObj11.addParameter(name='SNRthresh', value='0', format='int')
		56	# opObj11.addParameter(name='SNRmin', value='-10', format='int')
		57	# opObj11.addParameter(name='SNRmax', value='30', format='int')
		58
		59	# opObj11.addParameter(name='showSNR', value='1', format='int')
		60	# # opObj11.addParameter(name='channelList', value='0', format='intlist')
		61	# # opObj11.addParameter(name='xmin', value='0', format='float')
		62	# opObj11.addParameter(name='xmin', value='0', format='float')
		63	# opObj11.addParameter(name='xmax', value='24', format='float')
		64
		65	# opObj11.addParameter(name='zmin', value='-110', format='float')
		66	# opObj11.addParameter(name='zmax', value='-70', format='float')
		67	# opObj11.addParameter(name='save', value='0', format='int')
		68	# # opObj11.addParameter(name='figpath', value='/tmp/', format='str')
		69	#
		70	opObj12 = procUnitConfObj2.addOperation(name='PublishData', optype='other')
		71	opObj12.addParameter(name='zeromq', value=1, format='int')
		72	# opObj12.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str')
		73
		74
		75	# opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
		76	# opObj13.addParameter(name='zeromq', value=1, format='int')
		77	# opObj13.addParameter(name='server', value="juanca", format='str')
		78
		79	# opObj12.addParameter(name='delay', value=1, format='int')
		80
		81
		82	# print "Escribiendo el archivo XML"
		83	# controllerObj.writeXml(filename)
		84	# print "Leyendo el archivo XML"
		85	# controllerObj.readXml(filename)
		86
		87
		88	# timeit.timeit('controllerObj.run()', number=2)
65		89
66	controllerObj.start()	90	controllerObj.start()
67		91
68	if __name__=='__main__':
69
70	dt = datetime(2017, 1, 12)
71
72	dates = [(dt+timedelta(x)).strftime('%Y/%m/%d') for x in range(20)]
73		92
74	p = multiprocessing.Pool(4)	93	if __name__ == '__main__':
75	p.map(main, dates)	94	parser = argparse.ArgumentParser(description='Set number of parallel processes')
		95	parser.add_argument('--nProcess', default=1, type=int)
		96	args = parser.parse_args()
		97	multiSchain(fiber, nProcess=args.nProcess, startDate='2016/08/19', endDate='2016/08/19')

schainpy/scripts/receiver.py +18 -18

             #!/usr/bin/env python
             '''
             Created on Jul 7, 2014
             @author: roj-idl71
             '''
             import os, sys
             from schainpy.controller import Project
             if __name__ == '__main__':
                 desc = "Segundo Test"
                 controllerObj = Project()
                 controllerObj.setup(id='191', name='test01', description=desc)
                 proc1 = controllerObj.addProcUnit(name='ReceiverData')
                 proc1.addParameter(name='realtime', value='0', format='bool')
-                proc1.addParameter(name='plottypes', value='rti,coh,phase,snr,dop', format='str')
+                proc1.addParameter(name='plottypes', value='rti,coh,phase', format='str')
                 proc1.addParameter(name='throttle', value='10', format='int')
-                proc1.addParameter(name='plot_server', value='tcp://10.10.10.82:7000', format='str')
+                # proc1.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str')
                 ## TODO Agregar direccion de server de publicacion a graficos como variable
                 op1 = proc1.addOperation(name='PlotRTIData', optype='other')
                 op1.addParameter(name='wintitle', value='Julia 150Km', format='str')
                 op1.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
                 op1.addParameter(name='show', value='0', format='bool')
                 op1.addParameter(name='colormap', value='jet', format='str')
                 #
                 op2 = proc1.addOperation(name='PlotCOHData', optype='other')
                 op2.addParameter(name='wintitle', value='Julia 150Km', format='str')
                 op2.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
                 op2.addParameter(name='colormap', value='jet', format='str')
                 op2.addParameter(name='show', value='0', format='bool')
-                # #
+                # # #
                 op6 = proc1.addOperation(name='PlotPHASEData', optype='other')
                 op6.addParameter(name='wintitle', value='Julia 150Km', format='str')
                 op6.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
                 op6.addParameter(name='show', value='1', format='bool')
+                # #
-                # proc2 = controllerObj.addProcUnit(name='ReceiverData')
+                # # proc2 = controllerObj.addProcUnit(name='ReceiverData')
-                # proc2.addParameter(name='server', value='juanca', format='str')
+                # # proc2.addParameter(name='server', value='juanca', format='str')
-                # proc2.addParameter(name='plottypes', value='snr,dop', format='str')
+                # # proc2.addParameter(name='plottypes', value='snr,dop', format='str')
+                # #
-                op3 = proc1.addOperation(name='PlotSNRData', optype='other')
+                # op3 = proc1.addOperation(name='PlotSNRData', optype='other')
-                op3.addParameter(name='wintitle', value='Julia 150Km', format='str')
+                # op3.addParameter(name='wintitle', value='Julia 150Km', format='str')
-                op3.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
+                # op3.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
-                op3.addParameter(name='show', value='0', format='bool')
+                # op3.addParameter(name='show', value='0', format='bool')
+                # #
-                op4 = proc1.addOperation(name='PlotDOPData', optype='other')
+                # op4 = proc1.addOperation(name='PlotDOPData', optype='other')
-                op4.addParameter(name='wintitle', value='Julia 150Km', format='str')
+                # op4.addParameter(name='wintitle', value='Julia 150Km', format='str')
-                op4.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
+                # op4.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
-                op4.addParameter(name='show', value='0', format='bool')
+                # op4.addParameter(name='show', value='0', format='bool')
-                op4.addParameter(name='colormap', value='jet', format='str')
+                # op4.addParameter(name='colormap', value='jet', format='str')
                 controllerObj.start()

schainpy/scripts/schain.xml +1 -1

	@@ -1,1 +1,1
	1	<Project description="HF_EXAMPLE" id="191" name="test01"><ReadUnit datatype="SpectraReader" id="1911" inputId="0" name="SpectraReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="SpectraReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/data/~~julia~~" /><Parameter format="date" id="191113" name="startDate" value="201~~5/09/26~~" /><Parameter format="date" id="191114" name="endDate" value="201~~5/09/26~~" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="cursor" value="0" /><Parameter format="int" id="191119" name="skip" value="0" /><Parameter format="int" id="191120" name="~~delay~~" value="10" /><Parameter format="int" id="191121" name="~~walk~~" value="1" /><Parameter format="int" id="191122" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="ParametersProc" id="1913" inputId="1911" name="ParametersProc"><Operation id="19131" name="run" priority="1" type="self" /><Operation id="19132" name="SpectralMoments" priority="2" type="other" /><Operation id="19133" name="P~~arameters1Plot~~" priority="3" type="other"><Parameter format="int~~list~~" id="191331" name="~~channelList~~" value="0" /><Parameter format="int" id=~~"191332"~~ ~~name="id"~~ ~~value="2000"~~ ~~/><Parameter~~ ~~format="bool"~~ ~~id=~~"191333" name="~~onlySNR"~~ ~~value="1"~~ ~~/><Parameter~~ ~~format="bool"~~ ~~id="191334"~~ ~~name="DOP~~" value="0" ~~/></Operation><Operation~~ ~~id="19134"~~ ~~name="PublishData"~~ ~~priority="4"~~ ~~type="other"><Parameter~~ ~~format="int"~~ ~~id="191341"~~ ~~name="zeromq"~~ ~~value="1"~~ ~~/><Parameter~~ ~~format="int"~~ ~~id="191342"~~ ~~name="delay"~~ ~~value="1"~~ /></Operation></ProcUnit><ProcUnit datatype="Spectra" id="1912" inputId="1911" name="SpectraProc"><Operation id="19121" name="run" priority="1" type="self" /></ProcUnit></Project> No newline at end of file		1	<Project description="HF_EXAMPLE" id="191" name="test01"><ReadUnit datatype="SpectraReader" id="1911" inputId="0" name="SpectraReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="SpectraReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/data/sp1_f0" /><Parameter format="date" id="191113" name="startDate" value="2017/01/28" /><Parameter format="date" id="191114" name="endDate" value="2017/01/28" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="cursor" value="28" /><Parameter format="int" id="191119" name="skip" value="22" /><Parameter format="int" id="191120" name="walk" value="1" /><Parameter format="int" id="191121" name="verbose" value="1" /><Parameter format="int" id="191122" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="ParametersProc" id="1913" inputId="1911" name="ParametersProc"><Operation id="19131" name="run" priority="1" type="self" /><Operation id="19132" name="SpectralMoments" priority="2" type="other" /><Operation id="19133" name="PublishData" priority="3" type="other"><Parameter format="int" id="191331" name="zeromq" value="1" /><Parameter format="bool" id="191332" name="verbose" value="0" /><Parameter format="int" id="191333" name="delay" value="0" /></Operation></ProcUnit><ProcUnit datatype="Spectra" id="1912" inputId="1911" name="SpectraProc"><Operation id="19121" name="run" priority="1" type="self" /></ProcUnit></Project> No newline at end of file

setup.py +2 -1

             '''
             Created on Jul 16, 2014
             @author: Miguel Urco
             '''
             from schainpy import __version__
             from setuptools import setup, Extension
             setup(name="schainpy",
                     version=__version__,
                     description="Python tools to read, write and process Jicamarca data",
                     author="Miguel Urco",
                     author_email="miguel.urco@jro.igp.gob.pe",
                     url="http://jro.igp.gob.pe",
                     packages = {'schainpy',
                                 'schainpy.model',
                                 'schainpy.model.data',
                                 'schainpy.model.graphics',
                                 'schainpy.model.io',
                                 'schainpy.model.proc',
                                 'schainpy.model.serializer',
                                 'schainpy.model.utils',
                                 'schainpy.gui',
                                 'schainpy.gui.figures',
                                 'schainpy.gui.viewcontroller',
                                 'schainpy.gui.viewer',
                                 'schainpy.gui.viewer.windows'},
                     ext_package='schainpy',
                     py_modules=[''],
                     package_data={'': ['schain.conf.template'],
                                   'schainpy.gui.figures': ['*.png','*.jpg'],
                                   },
                     include_package_data=False,
                     scripts =['schainpy/gui/schainGUI',
                               'schainpy/scripts/schain'],
                     ext_modules=[Extension("cSchain", ["schainpy/model/proc/extensions.c"])],
                     install_requires=[
                                       "scipy >= 0.14.0",
                                       "h5py >= 2.2.1",
                                       "matplotlib >= 1.4.2",
                                       "pyfits >= 3.4",
                                       "numpy >= 1.11.2",
                                       "paramiko >= 2.1.2",
                                       "paho-mqtt >= 1.2",
                                       "zmq",
+                                      "fuzzywuzzy"
                                       ],
-                  )
  No newline at end of file

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages