schain Commit - r1279:c53fe2a4a291 · Jicamarca Repository

Modificación a kmamisr para ejecutarse en la versión 3, creación de scripts con terminación v3 para difereciarlos, se comentó la linea #720 de JroIO_param.py debido a que reiniciaba la lista de archivos, ocasionando la reescritura del archivo hdf5. Alguna otra modificación aparente es producto de algunas variaciones en espacios al usar la función print()

joabAM -

r1279:c53fe2a4a291

parent child

Context file:

r1279:c53fe2a4a291

Collapse all files

The requested changes are too big and content was truncated. Show full diff

schainpy/controller.py +41 -36

             '''
             Updated on January , 2018, for multiprocessing purposes
             Author: Sergio Cortez
             Created on September , 2012
             '''
             from platform import python_version
             import sys
             import ast
             import datetime
             import traceback
             import math
             import time
             import zmq
             from multiprocessing import Process, Queue, Event, Value, cpu_count
             from threading import Thread
             from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
             from xml.dom import minidom
             from schainpy.admin import Alarm, SchainWarning
             from schainpy.model import *
             from schainpy.utils import log
             DTYPES = {
                 'Voltage': '.r',
                 'Spectra': '.pdata'
             }
             def MPProject(project, n=cpu_count()):
                 '''
                 Project wrapper to run schain in n processes
                 '''
                 rconf = project.getReadUnitObj()
                 op = rconf.getOperationObj('run')
                 dt1 = op.getParameterValue('startDate')
                 dt2 = op.getParameterValue('endDate')
                 tm1 = op.getParameterValue('startTime')
                 tm2 = op.getParameterValue('endTime')
                 days = (dt2 - dt1).days
                 for day in range(days + 1):
                     skip = 0
                     cursor = 0
                     processes = []
                     dt = dt1 + datetime.timedelta(day)
                     dt_str = dt.strftime('%Y/%m/%d')
                     reader = JRODataReader()
                     paths, files = reader.searchFilesOffLine(path=rconf.path,
                                                              startDate=dt,
                                                              endDate=dt,
                                                              startTime=tm1,
                                                              endTime=tm2,
                                                              ext=DTYPES[rconf.datatype])
                     nFiles = len(files)
                     if nFiles == 0:
                         continue
                     skip = int(math.ceil(nFiles / n))
                     while nFiles > cursor * skip:
                         rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
                                      skip=skip)
                         p = project.clone()
                         p.start()
                         processes.append(p)
                         cursor += 1
                     def beforeExit(exctype, value, trace):
                         for process in processes:
                             process.terminate()
                             process.join()
                         print(traceback.print_tb(trace))
                     sys.excepthook = beforeExit
                     for process in processes:
                         process.join()
                         process.terminate()
                     time.sleep(3)
             def wait(context):
                 time.sleep(1)
                 c = zmq.Context()
                 receiver = c.socket(zmq.SUB)
                 receiver.connect('ipc:///tmp/schain_{}_pub'.format(self.id))
                 receiver.setsockopt(zmq.SUBSCRIBE, self.id.encode())
                 msg = receiver.recv_multipart()[1]
                 context.terminate()
             class ParameterConf():
                 id = None
                 name = None
                 value = None
                 format = None
                 __formated_value = None
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.format = 'str'
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getValue(self):
                     value = self.value
                     format = self.format
                     if self.__formated_value != None:
                         return self.__formated_value
                     if format == 'obj':
                         return value
                     if format == 'str':
                         self.__formated_value = str(value)
                         return self.__formated_value
                     if value == '':
                         raise ValueError('%s: This parameter value is empty' % self.name)
                     if format == 'list':
                         strList = [s.strip() for s in value.split(',')]
                         self.__formated_value = strList
                         return self.__formated_value
                     if format == 'intlist':
                         '''
                         Example:
                             value = (0,1,2)
                         '''
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             new_value = [int(new_value)]
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'floatlist':
                         '''
                         Example:
                             value = (0.5, 1.4, 2.7)
                         '''
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             new_value = [float(new_value)]
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'date':
                         strList = value.split('/')
                         intList = [int(x) for x in strList]
                         date = datetime.date(intList[0], intList[1], intList[2])
                         self.__formated_value = date
                         return self.__formated_value
                     if format == 'time':
                         strList = value.split(':')
                         intList = [int(x) for x in strList]
                         time = datetime.time(intList[0], intList[1], intList[2])
                         self.__formated_value = time
                         return self.__formated_value
                     if format == 'pairslist':
                         '''
                         Example:
                             value = (0,1),(1,2)
                         '''
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             raise ValueError('%s has to be a tuple or list of pairs' % value)
                         if type(new_value[0]) not in (tuple, list):
                             if len(new_value) != 2:
                                 raise ValueError('%s has to be a tuple or list of pairs' % value)
                             new_value = [new_value]
                         for thisPair in new_value:
                             if len(thisPair) != 2:
                                 raise ValueError('%s has to be a tuple or list of pairs' % value)
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'multilist':
                         '''
                         Example:
                             value = (0,1,2),(3,4,5)
                         '''
                         multiList = ast.literal_eval(value)
                         if type(multiList[0]) == int:
                             multiList = ast.literal_eval('(' + value + ')')
                         self.__formated_value = multiList
                         return self.__formated_value
                     if format == 'bool':
                         value = int(value)
                     if format == 'int':
                         value = float(value)
                     format_func = eval(format)
                     self.__formated_value = format_func(value)
                     return self.__formated_value
                 def updateId(self, new_id):
                     self.id = str(new_id)
                 def setup(self, id, name, value, format='str'):
                     self.id = str(id)
                     self.name = name
                     if format == 'obj':
                         self.value = value
                     else:
                         self.value = str(value)
                     self.format = str.lower(format)
                     self.getValue()
                     return 1
                 def update(self, name, value, format='str'):
                     self.name = name
                     self.value = str(value)
                     self.format = format
                 def makeXml(self, opElement):
                     if self.name not in ('queue',):
                         parmElement = SubElement(opElement, self.ELEMENTNAME)
                         parmElement.set('id', str(self.id))
                         parmElement.set('name', self.name)
                         parmElement.set('value', self.value)
                         parmElement.set('format', self.format)
                 def readXml(self, parmElement):
                     self.id = parmElement.get('id')
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                     self.format = str.lower(parmElement.get('format'))
                     # Compatible with old signal chain version
                     if self.format == 'int' and self.name == 'idfigure':
                         self.name = 'id'
                 def printattr(self):
                     print('Parameter[%s]: name = %s, value = %s, format = %s, project_id = %s' % (self.id, self.name, self.value, self.format, self.project_id))
             class OperationConf():
                 ELEMENTNAME = 'Operation'
                 def __init__(self):
                     self.id = '0'
                     self.name = None
                     self.priority = None
                     self.topic = None
                 def __getNewId(self):
                     return int(self.id) * 10 + len(self.parmConfObjList) + 1
                 def getId(self):
                     return self.id
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     n = 1
                     for parmObj in self.parmConfObjList:
                         idParm = str(int(new_id) * 10 + n)
                         parmObj.updateId(idParm)
                         n += 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getParameterObjList(self):
                     return self.parmConfObjList
                 def getParameterObj(self, parameterName):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.name != parameterName:
                             continue
                         return parmConfObj
                     return None
                 def getParameterObjfromValue(self, parameterValue):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.getValue() != parameterValue:
                             continue
                         return parmConfObj.getValue()
                     return None
                 def getParameterValue(self, parameterName):
                     parameterObj = self.getParameterObj(parameterName)
                 #         if not parameterObj:
                 #             return None
                     value = parameterObj.getValue()
                     return value
                 def getKwargs(self):
                     kwargs = {}
                     for parmConfObj in self.parmConfObjList:
                         if self.name == 'run' and parmConfObj.name == 'datatype':
                             continue
                         kwargs[parmConfObj.name] = parmConfObj.getValue()
                     return kwargs
                 def setup(self, id, name, priority, type, project_id, err_queue, lock):
                     self.id = str(id)
                     self.project_id = project_id
                     self.name = name
                     self.type = type
                     self.priority = priority
                     self.err_queue = err_queue
                     self.lock = lock
                     self.parmConfObjList = []
                 def removeParameters(self):
                     for obj in self.parmConfObjList:
                         del obj
                     self.parmConfObjList = []
                 def addParameter(self, name, value, format='str'):
                     if value is None:
                         return None
                     id = self.__getNewId()
                     parmConfObj = ParameterConf()
                     if not parmConfObj.setup(id, name, value, format):
                         return None
                     self.parmConfObjList.append(parmConfObj)
                     return parmConfObj
                 def changeParameter(self, name, value, format='str'):
                     parmConfObj = self.getParameterObj(name)
                     parmConfObj.update(name, value, format)
                     return parmConfObj
                 def makeXml(self, procUnitElement):
                     opElement = SubElement(procUnitElement, self.ELEMENTNAME)
                     opElement.set('id', str(self.id))
                     opElement.set('name', self.name)
                     opElement.set('type', self.type)
                     opElement.set('priority', str(self.priority))
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.makeXml(opElement)
                 def readXml(self, opElement, project_id):
                     self.id = opElement.get('id')
                     self.name = opElement.get('name')
                     self.type = opElement.get('type')
                     self.priority = opElement.get('priority')
                     self.project_id = str(project_id)
                     # Compatible with old signal chain version
                     # Use of 'run' method instead 'init'
                     if self.type == 'self' and self.name == 'init':
                         self.name = 'run'
                     self.parmConfObjList = []
                     parmElementList = opElement.iter(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         # Compatible with old signal chain version
                         # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
                         if self.type != 'self' and self.name == 'Plot':
                             if parmConfObj.format == 'str' and parmConfObj.name == 'type':
                                 self.name = parmConfObj.value
                                 continue
                         self.parmConfObjList.append(parmConfObj)
                 def printattr(self):
                     print('%s[%s]: name = %s, type = %s, priority = %s, project_id = %s' % (self.ELEMENTNAME,
                                                                            self.id,
                                                                            self.name,
                                                                            self.type,
                                                                            self.priority,
                                                                            self.project_id))
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.printattr()
                 def createObject(self):
                     className = eval(self.name)
                     if self.type == 'other':
                         opObj = className()
                     elif self.type == 'external':
                         kwargs = self.getKwargs()
                         opObj = className(self.id, self.id, self.project_id, self.err_queue, self.lock, 'Operation', **kwargs)
                         opObj.start()
                         self.opObj = opObj
                     return opObj
             class ProcUnitConf():
                 ELEMENTNAME = 'ProcUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.opConfObjList = []
                     self.procUnitObj = None
                     self.opObjDict = {}
                 def __getPriority(self):
                     return len(self.opConfObjList) + 1
                 def __getNewId(self):
                     return int(self.id) * 10 + len(self.opConfObjList) + 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def updateId(self, new_id):
                     '''
                     new_id = int(parentId) * 10 + (int(self.id) % 10)
                     new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
                     # If this proc unit has not inputs
                     #if self.inputId == '0':
                         #new_inputId = 0
                     n = 1
                     for opConfObj in self.opConfObjList:
                         idOp = str(int(new_id) * 10 + n)
                         opConfObj.updateId(idOp)
                         n += 1
                     self.parentId = str(parentId)
                     self.id = str(new_id)
                     #self.inputId = str(new_inputId)
                     '''
                     n = 1
                 def getInputId(self):
                     return self.inputId
                 def getOperationObjList(self):
                     return self.opConfObjList
                 def getOperationObj(self, name=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.name != name:
                             continue
                         return opConfObj
                     return None
                 def getOpObjfromParamValue(self, value=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
                             continue
                         return opConfObj
                     return None
                 def getProcUnitObj(self):
                     return self.procUnitObj
                 def setup(self, project_id, id, name, datatype, inputId, err_queue, lock):
                     '''
                         id sera el topico a publicar
                         inputId sera el topico a subscribirse
                     '''
                     # Compatible with old signal chain version
                     if datatype == None and name == None:
                         raise ValueError('datatype or name should be defined')
                     #Definir una condicion para inputId cuando sea 0
                     if name == None:
                         if 'Proc' in datatype:
                             name = datatype
                         else:
                             name = '%sProc' % (datatype)
                     if datatype == None:
                         datatype = name.replace('Proc', '')
                     self.id = str(id)
                     self.project_id = project_id
                     self.name = name
                     self.datatype = datatype
                     self.inputId = inputId
                     self.err_queue = err_queue
                     self.lock = lock
                     self.opConfObjList = []
                     self.addOperation(name='run', optype='self')
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                     self.addOperation(name='run')
                 def addParameter(self, **kwargs):
                     '''
                     Add parameters to 'run' operation
                     '''
                     opObj = self.opConfObjList[0]
                     opObj.addParameter(**kwargs)
                     return opObj
                 def addOperation(self, name, optype='self'):
                     '''
                         Actualizacion - > proceso comunicacion
                         En el caso de optype='self', elminar. DEfinir comuncacion IPC -> Topic
                         definir el tipoc de socket o comunicacion ipc++
                     '''
                     id = self.__getNewId()
                     priority = self.__getPriority() # Sin mucho sentido, pero puede usarse
                     opConfObj = OperationConf()
                     opConfObj.setup(id, name=name, priority=priority, type=optype, project_id=self.project_id, err_queue=self.err_queue, lock=self.lock)
                     self.opConfObjList.append(opConfObj)
                     return opConfObj
                 def makeXml(self, projectElement):
                     procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
                     procUnitElement.set('id', str(self.id))
                     procUnitElement.set('name', self.name)
                     procUnitElement.set('datatype', self.datatype)
                     procUnitElement.set('inputId', str(self.inputId))
                     for opConfObj in self.opConfObjList:
                         opConfObj.makeXml(procUnitElement)
                 def readXml(self, upElement, project_id):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.inputId = upElement.get('inputId')
                     self.project_id = str(project_id)
                     if self.ELEMENTNAME == 'ReadUnit':
                         self.datatype = self.datatype.replace('Reader', '')
                     if self.ELEMENTNAME == 'ProcUnit':
                         self.datatype = self.datatype.replace('Proc', '')
                     if self.inputId == 'None':
                         self.inputId = '0'
                     self.opConfObjList = []
                     opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement, project_id)
                         self.opConfObjList.append(opConfObj)
                 def printattr(self):
                     print('%s[%s]: name = %s, datatype = %s, inputId = %s, project_id = %s' % (self.ELEMENTNAME,
                                                                               self.id,
                                                                               self.name,
                                                                               self.datatype,
                                                                               self.inputId,
                                                                               self.project_id))
                     for opConfObj in self.opConfObjList:
                         opConfObj.printattr()
                 def getKwargs(self):
                     opObj = self.opConfObjList[0]
                     kwargs = opObj.getKwargs()
                     return kwargs
                 def createObjects(self):
                     '''
                     Instancia de unidades de procesamiento.
                     '''
                     className = eval(self.name)
+                    #print(self.name)
                     kwargs = self.getKwargs()
+                    #print(kwargs)
+                    #print("mark_a")
                     procUnitObj = className(self.id, self.inputId, self.project_id, self.err_queue, self.lock, 'ProcUnit', **kwargs)
+                    #print("mark_b")
                     log.success('creating process...', self.name)
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self' and opConfObj.name == 'run':
                             continue
                         elif opConfObj.type == 'self':
                             opObj = getattr(procUnitObj, opConfObj.name)
                         else:
                             opObj = opConfObj.createObject()
                         log.success('adding operation: {}, type:{}'.format(
                             opConfObj.name,
                             opConfObj.type), self.name)
                         procUnitObj.addOperation(opConfObj, opObj)
                     procUnitObj.start()
                     self.procUnitObj = procUnitObj
                 def close(self):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self':
                             continue
                         opObj = self.procUnitObj.getOperationObj(opConfObj.id)
                         opObj.close()
                     self.procUnitObj.close()
                     return
             class ReadUnitConf(ProcUnitConf):
                 ELEMENTNAME = 'ReadUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.opConfObjList = []
                     self.lock = Event()
                     self.lock.set()
                     self.lock.n = Value('d', 0)
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
                           startTime='', endTime='', server=None, **kwargs):
                     '''
                     *****el id del proceso sera el Topico
                     Adicion de {topic}, si no esta presente -> error
                     kwargs deben ser trasmitidos en la instanciacion
                     '''
                     # Compatible with old signal chain version
                     if datatype == None and name == None:
                         raise ValueError('datatype or name should be defined')
                     if name == None:
                         if 'Reader' in datatype:
                             name = datatype
                             datatype = name.replace('Reader','')
                         else:
                             name = '{}Reader'.format(datatype)
                     if datatype == None:
                         if 'Reader' in name:
                             datatype = name.replace('Reader','')
                         else:
                             datatype = name
                             name = '{}Reader'.format(name)
                     self.id = id
                     self.project_id = project_id
                     self.name = name
                     self.datatype = datatype
                     if path != '':
                         self.path = os.path.abspath(path)
+                    print (self.path)
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.server = server
                     self.err_queue = err_queue
                     self.addRunOperation(**kwargs)
                 def update(self, **kwargs):
                     if 'datatype' in kwargs:
                         datatype = kwargs.pop('datatype')
                         if 'Reader' in datatype:
                             self.name = datatype
                         else:
                             self.name = '%sReader' % (datatype)
                         self.datatype = self.name.replace('Reader', '')
                     attrs = ('path', 'startDate', 'endDate',
                              'startTime', 'endTime')
                     for attr in attrs:
                         if attr in kwargs:
                             setattr(self, attr, kwargs.pop(attr))
                     self.updateRunOperation(**kwargs)
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                 def addRunOperation(self, **kwargs):
                     opObj = self.addOperation(name='run', optype='self')
                     if self.server is None:
                         opObj.addParameter(
                             name='datatype', value=self.datatype, format='str')
                         opObj.addParameter(name='path', value=self.path, format='str')
                         opObj.addParameter(
                             name='startDate', value=self.startDate, format='date')
                         opObj.addParameter(
                             name='endDate', value=self.endDate, format='date')
                         opObj.addParameter(
                             name='startTime', value=self.startTime, format='time')
                         opObj.addParameter(
                             name='endTime', value=self.endTime, format='time')
                         for key, value in list(kwargs.items()):
                             opObj.addParameter(name=key, value=value,
                                                format=type(value).__name__)
                     else:
                         opObj.addParameter(name='server', value=self.server, format='str')
                     return opObj
                 def updateRunOperation(self, **kwargs):
                     opObj = self.getOperationObj(name='run')
                     opObj.removeParameters()
                     opObj.addParameter(name='datatype', value=self.datatype, format='str')
                     opObj.addParameter(name='path', value=self.path, format='str')
                     opObj.addParameter(
                         name='startDate', value=self.startDate, format='date')
                     opObj.addParameter(name='endDate', value=self.endDate, format='date')
                     opObj.addParameter(
                         name='startTime', value=self.startTime, format='time')
                     opObj.addParameter(name='endTime', value=self.endTime, format='time')
                     for key, value in list(kwargs.items()):
                         opObj.addParameter(name=key, value=value,
                                            format=type(value).__name__)
                     return opObj
                 def readXml(self, upElement, project_id):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.project_id = str(project_id)  #yong
                     if self.ELEMENTNAME == 'ReadUnit':
                         self.datatype = self.datatype.replace('Reader', '')
                     self.opConfObjList = []
                     opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement, project_id)
                         self.opConfObjList.append(opConfObj)
                         if opConfObj.name == 'run':
                             self.path = opConfObj.getParameterValue('path')
                             self.startDate = opConfObj.getParameterValue('startDate')
                             self.endDate = opConfObj.getParameterValue('endDate')
                             self.startTime = opConfObj.getParameterValue('startTime')
                             self.endTime = opConfObj.getParameterValue('endTime')
             class Project(Process):
                 ELEMENTNAME = 'Project'
                 def __init__(self):
                     Process.__init__(self)
                     self.id = None
                     self.filename = None
                     self.description = None
                     self.email = None
                     self.alarm = None
                     self.procUnitConfObjDict = {}
                     self.err_queue = Queue()
                 def __getNewId(self):
                     idList = list(self.procUnitConfObjDict.keys())
                     id = int(self.id) * 10
                     while True:
                         id += 1
                         if str(id) in idList:
                             continue
                         break
                     return str(id)
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     keyList = list(self.procUnitConfObjDict.keys())
                     keyList.sort()
                     n = 1
                     newProcUnitConfObjDict = {}
                     for procKey in keyList:
                         procUnitConfObj = self.procUnitConfObjDict[procKey]
                         idProcUnit = str(int(self.id) * 10 + n)
                         procUnitConfObj.updateId(idProcUnit)
                         newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
                         n += 1
                     self.procUnitConfObjDict = newProcUnitConfObjDict
                 def setup(self, id=1, name='', description='', email=None, alarm=[]):
                     print(' ')
                     print('*' * 60)
                     print('*  Starting SIGNAL CHAIN PROCESSING (Multiprocessing) v%s *' % schainpy.__version__)
                     print('*' * 60)
                     print("*  Python " + python_version() + "  *")
                     print('*' * 19)
                     print(' ')
                     self.id = str(id)
                     self.description = description
                     self.email = email
                     self.alarm = alarm
                     if name:
                         self.name = '{} ({})'.format(Process.__name__, name)
                 def update(self, **kwargs):
                     for key, value in list(kwargs.items()):
                         setattr(self, key, value)
                 def clone(self):
                     p = Project()
                     p.procUnitConfObjDict = self.procUnitConfObjDict
                     return p
                 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
                     '''
                     Actualizacion:
                         Se agrego un nuevo argumento: topic -relativo a la forma de comunicar los procesos simultaneos
                         * El id del proceso sera el topico al que se deben subscribir los procUnits para recibir la informacion(data)
                     '''
                     if id is None:
                         idReadUnit = self.__getNewId()
                     else:
                         idReadUnit = str(id)
                     readUnitConfObj = ReadUnitConf()
                     readUnitConfObj.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
                     self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     return readUnitConfObj
                 def addProcUnit(self, inputId='0', datatype=None, name=None):
                     '''
                     Actualizacion:
                         Se agrego dos nuevos argumentos: topic_read (lee data de otro procUnit) y topic_write(escribe o envia data a otro procUnit)
                         Deberia reemplazar a "inputId"
                         ** A fin de mantener el inputID, este sera la representaacion del topicoal que deben subscribirse. El ID propio de la intancia
                         (proceso) sera el topico de la publicacion, todo sera asignado de manera dinamica.
                     '''
                     idProcUnit = self.__getNewId()
                     procUnitConfObj = ProcUnitConf()
                     input_proc = self.procUnitConfObjDict[inputId]
                     procUnitConfObj.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue, input_proc.lock)
                     self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     return procUnitConfObj
                 def removeProcUnit(self, id):
                     if id in list(self.procUnitConfObjDict.keys()):
                         self.procUnitConfObjDict.pop(id)
                 def getReadUnitId(self):
                     readUnitConfObj = self.getReadUnitObj()
                     return readUnitConfObj.id
                 def getReadUnitObj(self):
                     for obj in list(self.procUnitConfObjDict.values()):
                         if obj.getElementName() == 'ReadUnit':
                             return obj
                     return None
                 def getProcUnitObj(self, id=None, name=None):
                     if id != None:
                         return self.procUnitConfObjDict[id]
                     if name != None:
                         return self.getProcUnitObjByName(name)
                     return None
                 def getProcUnitObjByName(self, name):
                     for obj in list(self.procUnitConfObjDict.values()):
                         if obj.name == name:
                             return obj
                     return None
                 def procUnitItems(self):
                     return list(self.procUnitConfObjDict.items())
                 def makeXml(self):
                     projectElement = Element('Project')
                     projectElement.set('id', str(self.id))
                     projectElement.set('name', self.name)
                     projectElement.set('description', self.description)
                     for procUnitConfObj in list(self.procUnitConfObjDict.values()):
                         procUnitConfObj.makeXml(projectElement)
                     self.projectElement = projectElement
                 def writeXml(self, filename=None):
                     if filename == None:
                         if self.filename:
                             filename = self.filename
                         else:
                             filename = 'schain.xml'
                     if not filename:
                         print('filename has not been defined. Use setFilename(filename) for do it.')
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.access(os.path.dirname(abs_file), os.W_OK):
                         print('No write permission on %s' % os.path.dirname(abs_file))
                         return 0
                     if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
                         print('File %s already exists and it could not be overwriten' % abs_file)
                         return 0
                     self.makeXml()
                     ElementTree(self.projectElement).write(abs_file, method='xml')
                     self.filename = abs_file
                     return 1
                 def readXml(self, filename=None):
                     if not filename:
                         print('filename is not defined')
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.path.isfile(abs_file):
                         print('%s file does not exist' % abs_file)
                         return 0
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     try:
                         self.projectElement = ElementTree().parse(abs_file)
                     except:
                         print('Error reading %s, verify file format' % filename)
                         return 0
                     self.project = self.projectElement.tag
                     self.id = self.projectElement.get('id')
                     self.name = self.projectElement.get('name')
                     self.description = self.projectElement.get('description')
                     readUnitElementList = self.projectElement.iter(
                         ReadUnitConf().getElementName())
                     for readUnitElement in readUnitElementList:
                         readUnitConfObj = ReadUnitConf()
                         readUnitConfObj.readXml(readUnitElement, self.id)
                         self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     procUnitElementList = self.projectElement.iter(
                         ProcUnitConf().getElementName())
                     for procUnitElement in procUnitElementList:
                         procUnitConfObj = ProcUnitConf()
                         procUnitConfObj.readXml(procUnitElement, self.id)
                         self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     self.filename = abs_file
                     return 1
                 def __str__(self):
                     print('Project: name = %s, description = %s, id = %s' % (
                                                                         self.name,
                                                                         self.description,
                                                                         self.id))
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         print(procUnitConfObj)
                 def createObjects(self):
                     keys = list(self.procUnitConfObjDict.keys())
                     keys.sort()
                     for key in keys:
                         self.procUnitConfObjDict[key].createObjects()
                 def monitor(self):
                     t = Thread(target=self.__monitor, args=(self.err_queue, self.ctx))
                     t.start()
                 def __monitor(self, queue, ctx):
                     import socket
                     procs = 0
                     err_msg = ''
                     while True:
                         msg = queue.get()
                         if '#_start_#' in msg:
                             procs += 1
                         elif '#_end_#' in msg:
                             procs -=1
                         else:
                             err_msg = msg
                         if procs == 0 or 'Traceback' in err_msg:
                             break
                         time.sleep(0.1)
                     if '|' in err_msg:
                         name, err = err_msg.split('|')
                         if 'SchainWarning' in err:
                             log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
                         elif 'SchainError' in err:
                             log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
                         else:
                             log.error(err, name)
                     else:
                         name, err = self.name, err_msg
                     time.sleep(2)
                     for conf in self.procUnitConfObjDict.values():
                         for confop in conf.opConfObjList:
                             if confop.type == 'external':
                                 confop.opObj.terminate()
                         conf.procUnitObj.terminate()
                     ctx.term()
                     message = ''.join(err)
                     if err_msg:
                         subject = 'SChain v%s: Error running %s\n' % (
                             schainpy.__version__, self.name)
                         subtitle = 'Hostname: %s\n' % socket.gethostbyname(
                             socket.gethostname())
                         subtitle += 'Working directory: %s\n' % os.path.abspath('./')
                         subtitle += 'Configuration file: %s\n' % self.filename
                         subtitle += 'Time: %s\n' % str(datetime.datetime.now())
                         readUnitConfObj = self.getReadUnitObj()
                         if readUnitConfObj:
                             subtitle += '\nInput parameters:\n'
                             subtitle += '[Data path = %s]\n' % readUnitConfObj.path
                             subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
                             subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
                             subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
                             subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
                             subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
                         a = Alarm(
                             modes=self.alarm,
                             email=self.email,
                             message=message,
                             subject=subject,
                             subtitle=subtitle,
                             filename=self.filename
                         )
                         a.start()
                 def isPaused(self):
                     return 0
                 def isStopped(self):
                     return 0
                 def runController(self):
                     '''
                     returns 0 when this process has been stopped, 1 otherwise
                     '''
                     if self.isPaused():
                         print('Process suspended')
                         while True:
                             time.sleep(0.1)
                             if not self.isPaused():
                                 break
                             if self.isStopped():
                                 break
                         print('Process reinitialized')
                     if self.isStopped():
                         print('Process stopped')
                         return 0
                     return 1
                 def setFilename(self, filename):
                     self.filename = filename
                 def setProxy(self):
                     if not os.path.exists('/tmp/schain'):
                         os.mkdir('/tmp/schain')
                     self.ctx = zmq.Context()
                     xpub = self.ctx.socket(zmq.XPUB)
                     xpub.bind('ipc:///tmp/schain/{}_pub'.format(self.id))
                     xsub = self.ctx.socket(zmq.XSUB)
                     xsub.bind('ipc:///tmp/schain/{}_sub'.format(self.id))
                     self.monitor()
                     try:
                         zmq.proxy(xpub, xsub)
                     except zmq.ContextTerminated:
                         xpub.close()
                         xsub.close()
                 def run(self):
                     log.success('Starting {}: {}'.format(self.name, self.id), tag='')
                     self.start_time = time.time()
                     self.createObjects()
                     self.setProxy()
                     log.success('{} Done (Time: {}s)'.format(
                         self.name,
                         time.time()-self.start_time), '')

schainpy/model/data/jrodata.py +13 -13

             '''
             $Author: murco $
             $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
             '''
             import copy
             import numpy
             import datetime
             import json
             from schainpy.utils import log
             from .jroheaderIO import SystemHeader, RadarControllerHeader
             def getNumpyDtype(dataTypeCode):
                 if dataTypeCode == 0:
                     numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
                 elif dataTypeCode == 1:
                     numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
                 elif dataTypeCode == 2:
                     numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
                 elif dataTypeCode == 3:
                     numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
                 elif dataTypeCode == 4:
                     numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                 elif dataTypeCode == 5:
                     numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
                 else:
                     raise ValueError('dataTypeCode was not defined')
                 return numpyDtype
             def getDataTypeCode(numpyDtype):
                 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
                     datatype = 0
                 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
                     datatype = 1
                 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
                     datatype = 2
                 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
                     datatype = 3
                 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
                     datatype = 4
                 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
                     datatype = 5
                 else:
                     datatype = None
                 return datatype
             def hildebrand_sekhon(data, navg):
                 """
                 This method is for the objective determination of the noise level in Doppler spectra. This
                 implementation technique is based on the fact that the standard deviation of the spectral
                 densities is equal to the mean spectral density for white Gaussian noise
                 Inputs:
                     Data    :    heights
                     navg    :    numbers of averages
                 Return:
                     mean    :    noise's level
                 """
                 sortdata = numpy.sort(data, axis=None)
                 lenOfData = len(sortdata)
                 nums_min = lenOfData*0.2
                 if nums_min <= 5:
                     nums_min = 5
                 sump = 0.
                 sumq = 0.
                 j = 0
                 cont = 1
                 while((cont == 1)and(j < lenOfData)):
                     sump += sortdata[j]
                     sumq += sortdata[j]**2
                     if j > nums_min:
                         rtest = float(j)/(j-1) + 1.0/navg
                         if ((sumq*j) > (rtest*sump**2)):
                             j = j - 1
                             sump = sump - sortdata[j]
                             sumq = sumq - sortdata[j]**2
                             cont = 0
                     j += 1
                 lnoise = sump / j
                 return lnoise
             class Beam:
                 def __init__(self):
                     self.codeList = []
                     self.azimuthList = []
                     self.zenithList = []
             class GenericData(object):
                 flagNoData = True
                 def copy(self, inputObj=None):
                     if inputObj == None:
                         return copy.deepcopy(self)
                     for key in list(inputObj.__dict__.keys()):
                         attribute = inputObj.__dict__[key]
                         # If this attribute is a tuple or list
                         if type(inputObj.__dict__[key]) in (tuple, list):
                             self.__dict__[key] = attribute[:]
                             continue
                         # If this attribute is another object or instance
                         if hasattr(attribute, '__dict__'):
                             self.__dict__[key] = attribute.copy()
                             continue
                         self.__dict__[key] = inputObj.__dict__[key]
                 def deepcopy(self):
                     return copy.deepcopy(self)
                 def isEmpty(self):
                     return self.flagNoData
             class JROData(GenericData):
                 #    m_BasicHeader = BasicHeader()
                 #    m_ProcessingHeader = ProcessingHeader()
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
             #    data = None
                 type = None
                 datatype = None  # dtype but in string
             #     dtype = None
             #    nChannels = None
             #    nHeights = None
                 nProfiles = None
                 heightList = None
                 channelList = None
                 flagDiscontinuousBlock = False
                 useLocalTime = False
                 utctime = None
                 timeZone = None
                 dstFlag = None
                 errorCount = None
                 blocksize = None
             #     nCode = None
             #     nBaud = None
             #     code = None
                 flagDecodeData = False  # asumo q la data no esta decodificada
                 flagDeflipData = False  # asumo q la data no esta sin flip
                 flagShiftFFT = False
             #     ippSeconds = None
             #     timeInterval = None
                 nCohInt = None
             #     noise = None
                 windowOfFilter = 1
                 # Speed of ligth
                 C = 3e8
                 frequency = 49.92e6
                 realtime = False
                 beacon_heiIndexList = None
                 last_block = None
                 blocknow = None
                 azimuth = None
                 zenith = None
                 beam = Beam()
                 profileIndex = None
                 error = None
                 data = None
                 nmodes = None
                 def __str__(self):
                     return '{} - {}'.format(self.type, self.getDatatime())
                 def getNoise(self):
                     raise NotImplementedError
                 def getNChannels(self):
                     return len(self.channelList)
                 def getChannelIndexList(self):
                     return list(range(self.nChannels))
                 def getNHeights(self):
                     return len(self.heightList)
                 def getHeiRange(self, extrapoints=0):
                     heis = self.heightList
             #        deltah = self.heightList[1] - self.heightList[0]
             #
             #        heis.append(self.heightList[-1])
                     return heis
                 def getDeltaH(self):
                     delta = self.heightList[1] - self.heightList[0]
                     return delta
                 def getltctime(self):
                     if self.useLocalTime:
                         return self.utctime - self.timeZone * 60
                     return self.utctime
                 def getDatatime(self):
                     datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
                     return datatimeValue
                 def getTimeRange(self):
                     datatime = []
                     datatime.append(self.ltctime)
                     datatime.append(self.ltctime + self.timeInterval + 1)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getFmaxTimeResponse(self):
                     period = (10**-6) * self.getDeltaH() / (0.15)
                     PRF = 1. / (period * self.nCohInt)
                     fmax = PRF
                     return fmax
                 def getFmax(self):
                     PRF = 1. / (self.ippSeconds * self.nCohInt)
                     fmax = PRF
                     return fmax
                 def getVmax(self):
                     _lambda = self.C / self.frequency
                     vmax = self.getFmax() * _lambda / 2
                     return vmax
                 def get_ippSeconds(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.ippSeconds
                 def set_ippSeconds(self, ippSeconds):
                     '''
                     '''
                     self.radarControllerHeaderObj.ippSeconds = ippSeconds
                     return
                 def get_dtype(self):
                     '''
                     '''
                     return getNumpyDtype(self.datatype)
                 def set_dtype(self, numpyDtype):
                     '''
                     '''
                     self.datatype = getDataTypeCode(numpyDtype)
                 def get_code(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.code
                 def set_code(self, code):
                     '''
                     '''
                     self.radarControllerHeaderObj.code = code
                     return
                 def get_ncode(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.nCode
                 def set_ncode(self, nCode):
                     '''
                     '''
                     self.radarControllerHeaderObj.nCode = nCode
                     return
                 def get_nbaud(self):
                     '''
                     '''
                     return self.radarControllerHeaderObj.nBaud
                 def set_nbaud(self, nBaud):
                     '''
                     '''
                     self.radarControllerHeaderObj.nBaud = nBaud
                     return
                 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
                 channelIndexList = property(
                     getChannelIndexList, "I'm the 'channelIndexList' property.")
                 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
                 #noise = property(getNoise, "I'm the 'nHeights' property.")
                 datatime = property(getDatatime, "I'm the 'datatime' property")
                 ltctime = property(getltctime, "I'm the 'ltctime' property")
                 ippSeconds = property(get_ippSeconds, set_ippSeconds)
                 dtype = property(get_dtype, set_dtype)
             #     timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
                 code = property(get_code, set_code)
                 nCode = property(get_ncode, set_ncode)
                 nBaud = property(get_nbaud, set_nbaud)
             class Voltage(JROData):
                 # data es un numpy array de 2 dmensiones (canales, alturas)
                 data = None
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.useLocalTime = True
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Voltage"
                     self.data = None
             #         self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
                     self.timeZone = None
                     self.dstFlag = None
                     self.errorCount = None
                     self.nCohInt = None
                     self.blocksize = None
                     self.flagDecodeData = False  # asumo q la data no esta decodificada
                     self.flagDeflipData = False  # asumo q la data no esta sin flip
                     self.flagShiftFFT = False
                     self.flagDataAsBlock = False  # Asumo que la data es leida perfil a perfil
                     self.profileIndex = 0
                 def getNoisebyHildebrand(self, channel=None):
                     """
                     Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                     Return:
                         noiselevel
                     """
                     if channel != None:
                         data = self.data[channel]
                         nChannels = 1
                     else:
                         data = self.data
                         nChannels = self.nChannels
                     noise = numpy.zeros(nChannels)
                     power = data * numpy.conjugate(data)
                     for thisChannel in range(nChannels):
                         if nChannels == 1:
                             daux = power[:].real
                         else:
                             daux = power[thisChannel, :].real
                         noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
                     return noise
                 def getNoise(self, type=1, channel=None):
                     if type == 1:
                         noise = self.getNoisebyHildebrand(channel)
                     return noise
                 def getPower(self, channel=None):
                     if channel != None:
                         data = self.data[channel]
                     else:
                         data = self.data
                     power = data * numpy.conjugate(data)
                     powerdB = 10 * numpy.log10(power.real)
                     powerdB = numpy.squeeze(powerdB)
                     return powerdB
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt
                     return timeInterval
                 noise = property(getNoise, "I'm the 'nHeights' property.")
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
             class Spectra(JROData):
                 # data spc es un numpy array de 2 dmensiones (canales, perfiles, alturas)
                 data_spc = None
                 # data cspc es un numpy array de 2 dmensiones (canales, pares, alturas)
                 data_cspc = None
                 # data dc es un numpy array de 2 dmensiones (canales, alturas)
                 data_dc = None
                 # data power
                 data_pwr = None
                 nFFTPoints = None
             #     nPairs = None
                 pairsList = None
                 nIncohInt = None
                 wavelength = None  # Necesario para cacular el rango de velocidad desde la frecuencia
                 nCohInt = None  # se requiere para determinar el valor de timeInterval
                 ippFactor = None
                 profileIndex = 0
                 plotting = "spectra"
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.useLocalTime = True
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Spectra"
             #        self.data = None
             #         self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.pairsList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
                     self.nCohInt = None
                     self.nIncohInt = None
                     self.blocksize = None
                     self.nFFTPoints = None
                     self.wavelength = None
                     self.flagDecodeData = False  # asumo q la data no esta decodificada
                     self.flagDeflipData = False  # asumo q la data no esta sin flip
                     self.flagShiftFFT = False
                     self.ippFactor = 1
                     #self.noise = None
                     self.beacon_heiIndexList = []
                     self.noise_estimation = None
                 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
                     """
                     Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
                     Return:
                         noiselevel
                     """
                     noise = numpy.zeros(self.nChannels)
                     for channel in range(self.nChannels):
                         daux = self.data_spc[channel,
                                              xmin_index:xmax_index, ymin_index:ymax_index]
                         noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
                     return noise
                 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
                     if self.noise_estimation is not None:
                         # this was estimated by getNoise Operation defined in jroproc_spectra.py
                         return self.noise_estimation
                     else:
                         noise = self.getNoisebyHildebrand(
                             xmin_index, xmax_index, ymin_index, ymax_index)
                         return noise
                 def getFreqRangeTimeResponse(self, extrapoints=0):
                     deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
                     freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
                     return freqrange
                 def getAcfRange(self, extrapoints=0):
                     deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
                     freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
                     return freqrange
                 def getFreqRange(self, extrapoints=0):
                     deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
                     freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
                     return freqrange
                 def getVelRange(self, extrapoints=0):
                     deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
                     velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
                     if self.nmodes:
                         return velrange/self.nmodes
                     else:
                         return velrange
                 def getNPairs(self):
                     return len(self.pairsList)
                 def getPairsIndexList(self):
                     return list(range(self.nPairs))
                 def getNormFactor(self):
                     pwcode = 1
                     if self.flagDecodeData:
                         pwcode = numpy.sum(self.code[0]**2)
                     #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
                     normFactor = self.nProfiles * self.nIncohInt * self.nCohInt * pwcode * self.windowOfFilter
                     return normFactor
                 def getFlagCspc(self):
                     if self.data_cspc is None:
                         return True
                     return False
                 def getFlagDc(self):
                     if self.data_dc is None:
                         return True
                     return False
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
                     if self.nmodes:
                         return self.nmodes*timeInterval
                     else:
                         return timeInterval
                 def getPower(self):
                     factor = self.normFactor
                     z = self.data_spc / factor
                     z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                     avg = numpy.average(z, axis=1)
                     return 10 * numpy.log10(avg)
                 def getCoherence(self, pairsList=None, phase=False):
                     z = []
                     if pairsList is None:
                         pairsIndexList = self.pairsIndexList
                     else:
                         pairsIndexList = []
                         for pair in pairsList:
                             if pair not in self.pairsList:
                                 raise ValueError("Pair %s is not in dataOut.pairsList" % (
                                     pair))
                             pairsIndexList.append(self.pairsList.index(pair))
                     for i in range(len(pairsIndexList)):
                         pair = self.pairsList[pairsIndexList[i]]
                         ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
                         powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
                         powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
                         avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
                         if phase:
                             data = numpy.arctan2(avgcoherenceComplex.imag,
                                                  avgcoherenceComplex.real) * 180 / numpy.pi
                         else:
                             data = numpy.abs(avgcoherenceComplex)
                         z.append(data)
                     return numpy.array(z)
                 def setValue(self, value):
                     print("This property should not be initialized")
                     return
                 nPairs = property(getNPairs, setValue, "I'm the 'nPairs' property.")
                 pairsIndexList = property(
                     getPairsIndexList, setValue, "I'm the 'pairsIndexList' property.")
                 normFactor = property(getNormFactor, setValue,
                                       "I'm the 'getNormFactor' property.")
                 flag_cspc = property(getFlagCspc, setValue)
                 flag_dc = property(getFlagDc, setValue)
                 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
                 timeInterval = property(getTimeInterval, setValue,
                                         "I'm the 'timeInterval' property")
             class SpectraHeis(Spectra):
                 data_spc = None
                 data_cspc = None
                 data_dc = None
                 nFFTPoints = None
             #     nPairs = None
                 pairsList = None
                 nCohInt = None
                 nIncohInt = None
                 def __init__(self):
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "SpectraHeis"
             #         self.dtype = None
             #        self.nChannels = 0
             #        self.nHeights = 0
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #        self.channelIndexList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
             #         self.nPairs = 0
                     self.utctime = None
                     self.blocksize = None
                     self.profileIndex = 0
                     self.nCohInt = 1
                     self.nIncohInt = 1
                 def getNormFactor(self):
                     pwcode = 1
                     if self.flagDecodeData:
                         pwcode = numpy.sum(self.code[0]**2)
                     normFactor = self.nIncohInt * self.nCohInt * pwcode
                     return normFactor
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     return timeInterval
                 normFactor = property(getNormFactor, "I'm the 'getNormFactor' property.")
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
             class Fits(JROData):
                 heightList = None
                 channelList = None
                 flagNoData = True
                 flagDiscontinuousBlock = False
                 useLocalTime = False
                 utctime = None
                 timeZone = None
             #     ippSeconds = None
             #     timeInterval = None
                 nCohInt = None
                 nIncohInt = None
                 noise = None
                 windowOfFilter = 1
                 # Speed of ligth
                 C = 3e8
                 frequency = 49.92e6
                 realtime = False
                 def __init__(self):
                     self.type = "Fits"
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
             #         self.channelIndexList = None
                     self.flagNoData = True
                     self.utctime = None
                     self.nCohInt = 1
                     self.nIncohInt = 1
                     self.useLocalTime = True
                     self.profileIndex = 0
             #         self.utctime = None
             #         self.timeZone = None
             #         self.ltctime = None
             #         self.timeInterval = None
             #         self.header = None
             #         self.data_header = None
             #         self.data = None
             #         self.datatime = None
             #         self.flagNoData = False
             #         self.expName = ''
             #         self.nChannels = None
             #         self.nSamples = None
             #         self.dataBlocksPerFile = None
             #         self.comments = ''
             #
                 def getltctime(self):
                     if self.useLocalTime:
                         return self.utctime - self.timeZone * 60
                     return self.utctime
                 def getDatatime(self):
                     datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
                     return datatime
                 def getTimeRange(self):
                     datatime = []
                     datatime.append(self.ltctime)
                     datatime.append(self.ltctime + self.timeInterval)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getHeiRange(self):
                     heis = self.heightList
                     return heis
                 def getNHeights(self):
                     return len(self.heightList)
                 def getNChannels(self):
                     return len(self.channelList)
                 def getChannelIndexList(self):
                     return list(range(self.nChannels))
                 def getNoise(self, type=1):
                     #noise = numpy.zeros(self.nChannels)
                     if type == 1:
                         noise = self.getNoisebyHildebrand()
                     if type == 2:
                         noise = self.getNoisebySort()
                     if type == 3:
                         noise = self.getNoisebyWindow()
                     return noise
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     return timeInterval
                 def get_ippSeconds(self):
                     '''
                     '''
                     return self.ipp_sec
                 datatime = property(getDatatime, "I'm the 'datatime' property")
                 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
                 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
                 channelIndexList = property(
                     getChannelIndexList, "I'm the 'channelIndexList' property.")
                 noise = property(getNoise, "I'm the 'nHeights' property.")
                 ltctime = property(getltctime, "I'm the 'ltctime' property")
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
                 ippSeconds = property(get_ippSeconds, '')
             class Correlation(JROData):
                 noise = None
                 SNR = None
                 #--------------------------------------------------
                 mode = None
                 split = False
                 data_cf = None
                 lags = None
                 lagRange = None
                 pairsList = None
                 normFactor = None
                 #--------------------------------------------------
             #     calculateVelocity = None
                 nLags = None
                 nPairs = None
                 nAvg = None
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Correlation"
                     self.data = None
                     self.dtype = None
                     self.nProfiles = None
                     self.heightList = None
                     self.channelList = None
                     self.flagNoData = True
                     self.flagDiscontinuousBlock = False
                     self.utctime = None
                     self.timeZone = None
                     self.dstFlag = None
                     self.errorCount = None
                     self.blocksize = None
                     self.flagDecodeData = False  # asumo q la data no esta decodificada
                     self.flagDeflipData = False  # asumo q la data no esta sin flip
                     self.pairsList = None
                     self.nPoints = None
                 def getPairsList(self):
                     return self.pairsList
                 def getNoise(self, mode=2):
                     indR = numpy.where(self.lagR == 0)[0][0]
                     indT = numpy.where(self.lagT == 0)[0][0]
                     jspectra0 = self.data_corr[:, :, indR, :]
                     jspectra = copy.copy(jspectra0)
                     num_chan = jspectra.shape[0]
                     num_hei = jspectra.shape[2]
                     freq_dc = jspectra.shape[1] / 2
                     ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
                     if ind_vel[0] < 0:
                         ind_vel[list(range(0, 1))] = ind_vel[list(
                             range(0, 1))] + self.num_prof
                     if mode == 1:
                         jspectra[:, freq_dc, :] = (
                             jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2  # CORRECCION
                     if mode == 2:
                         vel = numpy.array([-2, -1, 1, 2])
                         xx = numpy.zeros([4, 4])
                         for fil in range(4):
                             xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
                         xx_inv = numpy.linalg.inv(xx)
                         xx_aux = xx_inv[0, :]
                         for ich in range(num_chan):
                             yy = jspectra[ich, ind_vel, :]
                             jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
                             junkid = jspectra[ich, freq_dc, :] <= 0
                             cjunkid = sum(junkid)
                             if cjunkid.any():
                                 jspectra[ich, freq_dc, junkid.nonzero()] = (
                                     jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
                     noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
                     return noise
                 def getTimeInterval(self):
                     timeInterval = self.ippSeconds * self.nCohInt * self.nProfiles
                     return timeInterval
                 def splitFunctions(self):
                     pairsList = self.pairsList
                     ccf_pairs = []
                     acf_pairs = []
                     ccf_ind = []
                     acf_ind = []
                     for l in range(len(pairsList)):
                         chan0 = pairsList[l][0]
                         chan1 = pairsList[l][1]
                         # Obteniendo pares de Autocorrelacion
                         if chan0 == chan1:
                             acf_pairs.append(chan0)
                             acf_ind.append(l)
                         else:
                             ccf_pairs.append(pairsList[l])
                             ccf_ind.append(l)
                     data_acf = self.data_cf[acf_ind]
                     data_ccf = self.data_cf[ccf_ind]
                     return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
                 def getNormFactor(self):
                     acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
                     acf_pairs = numpy.array(acf_pairs)
                     normFactor = numpy.zeros((self.nPairs, self.nHeights))
                     for p in range(self.nPairs):
                         pair = self.pairsList[p]
                         ch0 = pair[0]
                         ch1 = pair[1]
                         ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
                         ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
                         normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
                     return normFactor
                 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
                 normFactor = property(getNormFactor, "I'm the 'normFactor property'")
             class Parameters(Spectra):
                 experimentInfo = None  # Information about the experiment
                 # Information from previous data
                 inputUnit = None  # Type of data to be processed
                 operation = None  # Type of operation to parametrize
                 # normFactor = None       #Normalization Factor
                 groupList = None  # List of Pairs, Groups, etc
                 # Parameters
                 data_param = None  # Parameters obtained
                 data_pre = None  # Data Pre Parametrization
                 data_SNR = None  # Signal to Noise Ratio
             #     heightRange = None      #Heights
                 abscissaList = None  # Abscissa, can be velocities, lags or time
             #    noise = None            #Noise Potency
                 utctimeInit = None  # Initial UTC time
                 paramInterval = None  # Time interval to calculate Parameters in seconds
                 useLocalTime = True
                 # Fitting
                 data_error = None  # Error of the estimation
                 constants = None
                 library = None
                 # Output signal
                 outputInterval = None  # Time interval to calculate output signal in seconds
                 data_output = None  # Out signal
                 nAvg = None
                 noise_estimation = None
                 GauSPC = None  # Fit gaussian SPC
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.systemHeaderObj = SystemHeader()
                     self.type = "Parameters"
                 def getTimeRange1(self, interval):
                     datatime = []
                     if self.useLocalTime:
                         time1 = self.utctimeInit - self.timeZone * 60
                     else:
                         time1 = self.utctimeInit
                     datatime.append(time1)
                     datatime.append(time1 + interval)
                     datatime = numpy.array(datatime)
                     return datatime
                 def getTimeInterval(self):
                     if hasattr(self, 'timeInterval1'):
                         return self.timeInterval1
                     else:
                         return self.paramInterval
                 def setValue(self, value):
                     print("This property should not be initialized")
                     return
                 def getNoise(self):
                     return self.spc_noise
                 timeInterval = property(getTimeInterval)
                 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
             class PlotterData(object):
                 '''
                 Object to hold data to be plotted
                 '''
                 MAXNUMX = 100
                 MAXNUMY = 100
                 def __init__(self, code, throttle_value, exp_code, buffering=True, snr=False):
                     self.key = code
                     self.throttle = throttle_value
                     self.exp_code = exp_code
                     self.buffering = buffering
                     self.ready = False
                     self.localtime = False
                     self.data = {}
                     self.meta = {}
                     self.__times = []
                     self.__heights = []
                     if 'snr' in code:
                         self.plottypes = ['snr']
                     elif code == 'spc':
                         self.plottypes = ['spc', 'noise', 'rti']
                     elif code == 'rti':
                         self.plottypes = ['noise', 'rti']
                     else:
                         self.plottypes = [code]
                     if 'snr' not in self.plottypes and snr:
                         self.plottypes.append('snr')
                     for plot in self.plottypes:
                         self.data[plot] = {}
                 def __str__(self):
                     dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
                     return 'Data[{}][{}]'.format(';'.join(dum), len(self.__times))
                 def __len__(self):
                     return len(self.__times)
                 def __getitem__(self, key):
                     if key not in self.data:
                         raise KeyError(log.error('Missing key: {}'.format(key)))
                     if 'spc' in key or not self.buffering:
                         ret = self.data[key]
                     elif 'scope' in key:
                         ret = numpy.array(self.data[key][float(self.tm)])
                     else:
                         ret = numpy.array([self.data[key][x] for x in self.times])
                         if ret.ndim > 1:
                             ret = numpy.swapaxes(ret, 0, 1)
                     return ret
                 def __contains__(self, key):
                     return key in self.data
                 def setup(self):
                     '''
                     Configure object
                     '''
                     self.type = ''
                     self.ready = False
                     self.data = {}
                     self.__times = []
                     self.__heights = []
                     self.__all_heights = set()
                     for plot in self.plottypes:
                         if 'snr' in plot:
                             plot = 'snr'
                         elif 'spc_moments' == plot:
                             plot = 'moments'
                         self.data[plot] = {}
                     if 'spc' in self.data or 'rti' in self.data or 'cspc' in self.data or 'moments' in self.data:
                         self.data['noise'] = {}
                         self.data['rti'] = {}
                         if 'noise' not in self.plottypes:
                             self.plottypes.append('noise')
                         if 'rti' not in self.plottypes:
                             self.plottypes.append('rti')
                 def shape(self, key):
                     '''
                     Get the shape of the one-element data for the given key
                     '''
                     if len(self.data[key]):
                         if 'spc' in key or not self.buffering:
                             return self.data[key].shape
                         return self.data[key][self.__times[0]].shape
                     return (0,)
                 def update(self, dataOut, tm):
                     '''
                     Update data object with new dataOut
                     '''
                     if tm in self.__times:
                         return
                     self.profileIndex = dataOut.profileIndex
                     self.tm = tm
                     self.type = dataOut.type
                     self.parameters = getattr(dataOut, 'parameters', [])
                     if hasattr(dataOut, 'meta'):
                         self.meta.update(dataOut.meta)
                     self.pairs = dataOut.pairsList
                     self.interval = dataOut.getTimeInterval()
                     self.localtime = dataOut.useLocalTime
                     if 'spc' in self.plottypes or 'cspc' in self.plottypes or 'spc_moments' in self.plottypes:
                         self.xrange = (dataOut.getFreqRange(1)/1000.,
                                        dataOut.getAcfRange(1), dataOut.getVelRange(1))
                         self.factor = dataOut.normFactor
                     self.__heights.append(dataOut.heightList)
                     self.__all_heights.update(dataOut.heightList)
                     self.__times.append(tm)
                     for plot in self.plottypes:
                         if plot in ('spc', 'spc_moments'):
                             z = dataOut.data_spc/dataOut.normFactor
                             buffer = 10*numpy.log10(z)
                         if plot == 'cspc':
                             z = dataOut.data_spc/dataOut.normFactor
                             buffer = (dataOut.data_spc, dataOut.data_cspc)
                         if plot == 'noise':
                             buffer = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
                         if plot == 'rti':
                             buffer = dataOut.getPower()
                         if plot == 'snr_db':
                             buffer = dataOut.data_SNR
                         if plot == 'snr':
                             buffer = 10*numpy.log10(dataOut.data_SNR)
                         if plot == 'dop':
                             buffer = dataOut.data_DOP
                         if plot == 'pow':
                             buffer = 10*numpy.log10(dataOut.data_POW)
                         if plot == 'width':
                             buffer = dataOut.data_WIDTH
                         if plot == 'coh':
                             buffer = dataOut.getCoherence()
                         if plot == 'phase':
                             buffer = dataOut.getCoherence(phase=True)
                         if plot == 'output':
                             buffer = dataOut.data_output
                         if plot == 'param':
                             buffer = dataOut.data_param
                         if plot == 'scope':
                             buffer = dataOut.data
                             self.flagDataAsBlock = dataOut.flagDataAsBlock
                             self.nProfiles = dataOut.nProfiles
                         if plot == 'spc':
                             self.data['spc'] = buffer
                         elif plot == 'cspc':
                             self.data['spc'] = buffer[0]
                             self.data['cspc'] = buffer[1]
                         elif plot == 'spc_moments':
                             self.data['spc'] = buffer
                             self.data['moments'][tm] = dataOut.moments
                         else:
                             if self.buffering:
                                 self.data[plot][tm] = buffer
                             else:
                                 self.data[plot] = buffer
                     if dataOut.channelList is None:
                         self.channels = range(buffer.shape[0])
                     else:
                         self.channels = dataOut.channelList
                 def normalize_heights(self):
                     '''
                     Ensure same-dimension of the data for different heighList
                     '''
                     H = numpy.array(list(self.__all_heights))
                     H.sort()
                     for key in self.data:
                         shape = self.shape(key)[:-1] + H.shape
                         for tm, obj in list(self.data[key].items()):
                             h = self.__heights[self.__times.index(tm)]
                             if H.size == h.size:
                                 continue
                             index = numpy.where(numpy.in1d(H, h))[0]
                             dummy = numpy.zeros(shape) + numpy.nan
                             if len(shape) == 2:
                                 dummy[:, index] = obj
                             else:
                                 dummy[index] = obj
                             self.data[key][tm] = dummy
                     self.__heights = [H for tm in self.__times]
                 def jsonify(self, plot_name, plot_type, decimate=False):
                     '''
                     Convert data to json
                     '''
                     tm = self.times[-1]
                     dy = int(self.heights.size/self.MAXNUMY) + 1
                     if self.key in ('spc', 'cspc') or not self.buffering:
                         dx = int(self.data[self.key].shape[1]/self.MAXNUMX) + 1
                         data = self.roundFloats(
                             self.data[self.key][::, ::dx, ::dy].tolist())
                     else:
                         data = self.roundFloats(self.data[self.key][tm].tolist())
                         if self.key is 'noise':
                             data = [[x] for x in data]
                     meta = {}
                     ret = {
                         'plot': plot_name,
                         'code': self.exp_code,
                         'time': float(tm),
                         'data': data,
                         }
                     meta['type'] = plot_type
                     meta['interval'] = float(self.interval)
                     meta['localtime'] = self.localtime
                     meta['yrange'] = self.roundFloats(self.heights[::dy].tolist())
                     if 'spc' in self.data or 'cspc' in self.data:
                         meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
                     else:
                         meta['xrange'] = []
                     meta.update(self.meta)
                     ret['metadata'] = meta
                     return json.dumps(ret)
                 @property
                 def times(self):
                     '''
                     Return the list of times of the current data
                     '''
                     ret = numpy.array(self.__times)
                     ret.sort()
                     return ret
                 @property
                 def min_time(self):
                     '''
                     Return the minimun time value
                     '''
                     return self.times[0]
                 @property
                 def max_time(self):
                     '''
                     Return the maximun time value
                     '''
                     return self.times[-1]
                 @property
                 def heights(self):
                     '''
                     Return the list of heights of the current data
                     '''
                     return numpy.array(self.__heights[-1])
                 @staticmethod
                 def roundFloats(obj):
                     if isinstance(obj, list):
                         return list(map(PlotterData.roundFloats, obj))
                     elif isinstance(obj, float):
                         return round(obj, 2)

schainpy/model/data/jroheaderIO.py +2 -2

             '''
             $Author: murco $
             $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
             '''
             import sys
             import numpy
             import copy
             import datetime
             import inspect
             from schainpy.utils import log
             SPEED_OF_LIGHT = 299792458
             SPEED_OF_LIGHT = 3e8
             BASIC_STRUCTURE = numpy.dtype([
                 ('nSize', '<u4'),
                 ('nVersion', '<u2'),
                 ('nDataBlockId', '<u4'),
                 ('nUtime', '<u4'),
                 ('nMilsec', '<u2'),
                 ('nTimezone', '<i2'),
                 ('nDstflag', '<i2'),
                 ('nErrorCount', '<u4')
             ])
             SYSTEM_STRUCTURE = numpy.dtype([
                 ('nSize', '<u4'),
                 ('nNumSamples', '<u4'),
                 ('nNumProfiles', '<u4'),
                 ('nNumChannels', '<u4'),
                 ('nADCResolution', '<u4'),
                 ('nPCDIOBusWidth', '<u4'),
             ])
             RADAR_STRUCTURE = numpy.dtype([
                                          ('nSize', '<u4'),
                                          ('nExpType', '<u4'),
                                          ('nNTx', '<u4'),
                                          ('fIpp', '<f4'),
                                          ('fTxA', '<f4'),
                                          ('fTxB', '<f4'),
                                          ('nNumWindows', '<u4'),
                                          ('nNumTaus', '<u4'),
                                          ('nCodeType', '<u4'),
                                          ('nLine6Function', '<u4'),
                                          ('nLine5Function', '<u4'),
                                          ('fClock', '<f4'),
                                          ('nPrePulseBefore', '<u4'),
                                          ('nPrePulseAfter', '<u4'),
                                          ('sRangeIPP', '<a20'),
                                          ('sRangeTxA', '<a20'),
                                          ('sRangeTxB', '<a20'),
             ])
             SAMPLING_STRUCTURE = numpy.dtype(
                 [('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')])
             PROCESSING_STRUCTURE = numpy.dtype([
                 ('nSize', '<u4'),
                 ('nDataType', '<u4'),
                 ('nSizeOfDataBlock', '<u4'),
                 ('nProfilesperBlock', '<u4'),
                 ('nDataBlocksperFile', '<u4'),
                 ('nNumWindows', '<u4'),
                 ('nProcessFlags', '<u4'),
                 ('nCoherentIntegrations', '<u4'),
                 ('nIncoherentIntegrations', '<u4'),
                 ('nTotalSpectra', '<u4')
             ])
             class Header(object):
                 def __init__(self):
                     raise NotImplementedError
                 def copy(self):
                     return copy.deepcopy(self)
                 def read(self):
                     raise NotImplementedError
                 def write(self):
                     raise NotImplementedError
                 def getAllowedArgs(self):
                     args = inspect.getargspec(self.__init__).args
                     try:
                         args.remove('self')
                     except:
                         pass
                     return args
                 def getAsDict(self):
                     args = self.getAllowedArgs()
                     asDict = {}
                     for x in args:
                         asDict[x] = self[x]
                     return asDict
                 def __getitem__(self, name):
                     return getattr(self, name)
                 def printInfo(self):
                     message = "#" * 50 + "\n"
                     message += self.__class__.__name__.upper() + "\n"
                     message += "#" * 50 + "\n"
                     keyList = list(self.__dict__.keys())
                     keyList.sort()
                     for key in keyList:
                         message += "%s = %s" % (key, self.__dict__[key]) + "\n"
                     if "size" not in keyList:
                         attr = getattr(self, "size")
                         if attr:
                             message += "%s = %s" % ("size", attr) + "\n"
                     print(message)
             class BasicHeader(Header):
                 size = None
                 version = None
                 dataBlock = None
                 utc = None
                 ltc = None
                 miliSecond = None
                 timeZone = None
                 dstFlag = None
                 errorCount = None
                 datatime = None
                 structure = BASIC_STRUCTURE
                 __LOCALTIME = None
                 def __init__(self, useLocalTime=True):
                     self.size = 24
                     self.version = 0
                     self.dataBlock = 0
                     self.utc = 0
                     self.miliSecond = 0
                     self.timeZone = 0
                     self.dstFlag = 0
                     self.errorCount = 0
                     self.useLocalTime = useLocalTime
                 def read(self, fp):
                     self.length = 0
                     try:
                         if hasattr(fp, 'read'):
                             header = numpy.fromfile(fp, BASIC_STRUCTURE, 1)
                         else:
                             header = numpy.fromstring(fp, BASIC_STRUCTURE, 1)
                     except Exception as e:
                         print("BasicHeader: ")
                         print(e)
                         return 0
                     self.size = int(header['nSize'][0])
                     self.version = int(header['nVersion'][0])
                     self.dataBlock = int(header['nDataBlockId'][0])
                     self.utc = int(header['nUtime'][0])
                     self.miliSecond = int(header['nMilsec'][0])
                     self.timeZone = int(header['nTimezone'][0])
                     self.dstFlag = int(header['nDstflag'][0])
                     self.errorCount = int(header['nErrorCount'][0])
                     if self.size < 24:
                         return 0
                     self.length = header.nbytes
                     return 1
                 def write(self, fp):
                     headerTuple = (self.size, self.version, self.dataBlock, self.utc,
                                    self.miliSecond, self.timeZone, self.dstFlag, self.errorCount)
                     header = numpy.array(headerTuple, BASIC_STRUCTURE)
                     header.tofile(fp)
                     return 1
                 def get_ltc(self):
                     return self.utc - self.timeZone * 60
                 def set_ltc(self, value):
                     self.utc = value + self.timeZone * 60
                 def get_datatime(self):
                     return datetime.datetime.utcfromtimestamp(self.ltc)
                 ltc = property(get_ltc, set_ltc)
                 datatime = property(get_datatime)
             class SystemHeader(Header):
                 size = None
                 nSamples = None
                 nProfiles = None
                 nChannels = None
                 adcResolution = None
                 pciDioBusWidth = None
                 structure = SYSTEM_STRUCTURE
                 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWidth=0):
                     self.size = 24
                     self.nSamples = nSamples
                     self.nProfiles = nProfiles
                     self.nChannels = nChannels
                     self.adcResolution = adcResolution
                     self.pciDioBusWidth = pciDioBusWidth
                 def read(self, fp):
                     self.length = 0
                     try:
                         startFp = fp.tell()
                     except Exception as e:
                         startFp = None
                         pass
                     try:
                         if hasattr(fp, 'read'):
                             header = numpy.fromfile(fp, SYSTEM_STRUCTURE, 1)
                         else:
                             header = numpy.fromstring(fp, SYSTEM_STRUCTURE, 1)
                     except Exception as e:
                         print("System Header: " + str(e))
                         return 0
                     self.size = header['nSize'][0]
                     self.nSamples = header['nNumSamples'][0]
                     self.nProfiles = header['nNumProfiles'][0]
                     self.nChannels = header['nNumChannels'][0]
                     self.adcResolution = header['nADCResolution'][0]
                     self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
                     if startFp is not None:
                         endFp = self.size + startFp
                         if fp.tell() > endFp:
                             sys.stderr.write(
                                 "Warning %s: Size value read from System Header is lower than it has to be\n" % fp.name)
                             return 0
                         if fp.tell() < endFp:
                             sys.stderr.write(
                                 "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp.name)
                             return 0
                     self.length = header.nbytes
                     return 1
                 def write(self, fp):
                     headerTuple = (self.size, self.nSamples, self.nProfiles,
                                    self.nChannels, self.adcResolution, self.pciDioBusWidth)
                     header = numpy.array(headerTuple, SYSTEM_STRUCTURE)
                     header.tofile(fp)
                     return 1
             class RadarControllerHeader(Header):
                 expType = None
                 nTx = None
                 ipp = None
                 txA = None
                 txB = None
                 nWindows = None
                 numTaus = None
                 codeType = None
                 line6Function = None
                 line5Function = None
                 fClock = None
                 prePulseBefore = None
                 prePulseAfter = None
                 rangeIpp = None
                 rangeTxA = None
                 rangeTxB = None
                 structure = RADAR_STRUCTURE
                 __size = None
                 def __init__(self, expType=2, nTx=1,
                              ipp=None, txA=0, txB=0,
                              nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
                              numTaus=0, line6Function=0, line5Function=0, fClock=None,
                              prePulseBefore=0, prePulseAfter=0,
                              codeType=0, nCode=0, nBaud=0, code=None,
                              flip1=0, flip2=0):
                     #         self.size = 116
                     self.expType = expType
                     self.nTx = nTx
                     self.ipp = ipp
                     self.txA = txA
                     self.txB = txB
                     self.rangeIpp = ipp
                     self.rangeTxA = txA
                     self.rangeTxB = txB
                     self.nWindows = nWindows
                     self.numTaus = numTaus
                     self.codeType = codeType
                     self.line6Function = line6Function
                     self.line5Function = line5Function
                     self.fClock = fClock
                     self.prePulseBefore = prePulseBefore
                     self.prePulseAfter = prePulseAfter
                     self.nHeights = nHeights
                     self.firstHeight = firstHeight
                     self.deltaHeight = deltaHeight
                     self.samplesWin = nHeights
                     self.nCode = nCode
                     self.nBaud = nBaud
                     self.code = code
                     self.flip1 = flip1
                     self.flip2 = flip2
                     self.code_size = int(numpy.ceil(self.nBaud / 32.)) * self.nCode * 4
             #         self.dynamic = numpy.array([],numpy.dtype('byte'))
                     if self.fClock is None and self.deltaHeight is not None:
                         self.fClock = 0.15 / (deltaHeight * 1e-6)  # 0.15Km / (height * 1u)
                 def read(self, fp):
                     self.length = 0
                     try:
                         startFp = fp.tell()
                     except Exception as e:
                         startFp = None
                         pass
                     try:
                         if hasattr(fp, 'read'):
                             header = numpy.fromfile(fp, RADAR_STRUCTURE, 1)
                         else:
                             header = numpy.fromstring(fp, RADAR_STRUCTURE, 1)
                         self.length += header.nbytes
                     except Exception as e:
                         print("RadarControllerHeader: " + str(e))
                         return 0
                     size = int(header['nSize'][0])
                     self.expType = int(header['nExpType'][0])
                     self.nTx = int(header['nNTx'][0])
                     self.ipp = float(header['fIpp'][0])
                     self.txA = float(header['fTxA'][0])
                     self.txB = float(header['fTxB'][0])
                     self.nWindows = int(header['nNumWindows'][0])
                     self.numTaus = int(header['nNumTaus'][0])
                     self.codeType = int(header['nCodeType'][0])
                     self.line6Function = int(header['nLine6Function'][0])
                     self.line5Function = int(header['nLine5Function'][0])
                     self.fClock = float(header['fClock'][0])
                     self.prePulseBefore = int(header['nPrePulseBefore'][0])
                     self.prePulseAfter = int(header['nPrePulseAfter'][0])
                     self.rangeIpp = header['sRangeIPP'][0]
                     self.rangeTxA = header['sRangeTxA'][0]
                     self.rangeTxB = header['sRangeTxB'][0]
                     try:
                         if hasattr(fp, 'read'):
                             samplingWindow = numpy.fromfile(
                                 fp, SAMPLING_STRUCTURE, self.nWindows)
                         else:
                             samplingWindow = numpy.fromstring(
                                 fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
                         self.length += samplingWindow.nbytes
                     except Exception as e:
                         print("RadarControllerHeader: " + str(e))
                         return 0
                     self.nHeights = int(numpy.sum(samplingWindow['nsa']))
                     self.firstHeight = samplingWindow['h0']
                     self.deltaHeight = samplingWindow['dh']
                     self.samplesWin = samplingWindow['nsa']
                     try:
                         if hasattr(fp, 'read'):
                             self.Taus = numpy.fromfile(fp, '<f4', self.numTaus)
                         else:
                             self.Taus = numpy.fromstring(
                                 fp[self.length:], '<f4', self.numTaus)
                         self.length += self.Taus.nbytes
                     except Exception as e:
                         print("RadarControllerHeader: " + str(e))
                         return 0
                     self.code_size = 0
                     if self.codeType != 0:
                         try:
                             if hasattr(fp, 'read'):
                                 self.nCode = numpy.fromfile(fp, '<u4', 1)[0]
                                 self.length += self.nCode.nbytes
                                 self.nBaud = numpy.fromfile(fp, '<u4', 1)[0]
                                 self.length += self.nBaud.nbytes
                             else:
                                 self.nCode = numpy.fromstring(
                                     fp[self.length:], '<u4', 1)[0]
                                 self.length += self.nCode.nbytes
                                 self.nBaud = numpy.fromstring(
                                     fp[self.length:], '<u4', 1)[0]
                                 self.length += self.nBaud.nbytes
                         except Exception as e:
                             print("RadarControllerHeader: " + str(e))
                             return 0
                         code = numpy.empty([self.nCode, self.nBaud], dtype='i1')
                         for ic in range(self.nCode):
                             try:
                                 if hasattr(fp, 'read'):
                                     temp = numpy.fromfile(fp, 'u4', int(
                                         numpy.ceil(self.nBaud / 32.)))
                                 else:
                                     temp = numpy.fromstring(
                                         fp, 'u4', int(numpy.ceil(self.nBaud / 32.)))
                                 self.length += temp.nbytes
                             except Exception as e:
                                 print("RadarControllerHeader: " + str(e))
                                 return 0
                             for ib in range(self.nBaud - 1, -1, -1):
                                 code[ic, ib] = temp[int(ib / 32)] % 2
                                 temp[int(ib / 32)] = temp[int(ib / 32)] / 2
                         self.code = 2.0 * code - 1.0
                         self.code_size = int(numpy.ceil(self.nBaud / 32.)) * self.nCode * 4
             #             if self.line5Function == RCfunction.FLIP:
             #                 self.flip1 = numpy.fromfile(fp,'<u4',1)
             #
             #             if self.line6Function == RCfunction.FLIP:
             #                 self.flip2 = numpy.fromfile(fp,'<u4',1)
                     if startFp is not None:
                         endFp = size + startFp
                         if fp.tell() != endFp:
                             #             fp.seek(endFp)
                             print("%s: Radar Controller Header size is not consistent: from data [%d] != from header field [%d]" % (fp.name, fp.tell() - startFp, size))
                 #             return 0
                         if fp.tell() > endFp:
                             sys.stderr.write(
                                 "Warning %s: Size value read from Radar Controller header is lower than it has to be\n" % fp.name)
                 #             return 0
                         if fp.tell() < endFp:
                             sys.stderr.write(
                                 "Warning %s: Size value read from Radar Controller header is greater than it has to be\n" % fp.name)
                     return 1
                 def write(self, fp):
                     headerTuple = (self.size,
                                    self.expType,
                                    self.nTx,
                                    self.ipp,
                                    self.txA,
                                    self.txB,
                                    self.nWindows,
                                    self.numTaus,
                                    self.codeType,
                                    self.line6Function,
                                    self.line5Function,
                                    self.fClock,
                                    self.prePulseBefore,
                                    self.prePulseAfter,
                                    self.rangeIpp,
                                    self.rangeTxA,
                                    self.rangeTxB)
                     header = numpy.array(headerTuple, RADAR_STRUCTURE)
                     header.tofile(fp)
                     sampleWindowTuple = (
                         self.firstHeight, self.deltaHeight, self.samplesWin)
                     samplingWindow = numpy.array(sampleWindowTuple, SAMPLING_STRUCTURE)
                     samplingWindow.tofile(fp)
                     if self.numTaus > 0:
                         self.Taus.tofile(fp)
                     if self.codeType != 0:
                         nCode = numpy.array(self.nCode, '<u4')
                         nCode.tofile(fp)
                         nBaud = numpy.array(self.nBaud, '<u4')
                         nBaud.tofile(fp)
                         code1 = (self.code + 1.0) / 2.
                         for ic in range(self.nCode):
                             tempx = numpy.zeros(int(numpy.ceil(self.nBaud / 32.)))
                             start = 0
                             end = 32
                             for i in range(len(tempx)):
                                 code_selected = code1[ic, start:end]
                                 for j in range(len(code_selected) - 1, -1, -1):
                                     if code_selected[j] == 1:
                                         tempx[i] = tempx[i] + \
 **(len(code_selected) - 1 - j)
                                 start = start + 32
                                 end = end + 32
                             tempx = tempx.astype('u4')
                             tempx.tofile(fp)
             #         if self.line5Function == RCfunction.FLIP:
             #             self.flip1.tofile(fp)
             #
             #         if self.line6Function == RCfunction.FLIP:
             #             self.flip2.tofile(fp)
                     return 1
                 def get_ippSeconds(self):
                     '''
                     '''
                     ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT
                     return ippSeconds
                 def set_ippSeconds(self, ippSeconds):
                     '''
                     '''
                     self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0 * 1000)
                     return
                 def get_size(self):
                     self.__size = 116 + 12 * self.nWindows + 4 * self.numTaus
                     if self.codeType != 0:
                         self.__size += 4 + 4 + 4 * self.nCode * \
                             numpy.ceil(self.nBaud / 32.)
                     return self.__size
                 def set_size(self, value):
                     raise IOError("size is a property and it cannot be set, just read")
                     return
                 ippSeconds = property(get_ippSeconds, set_ippSeconds)
                 size = property(get_size, set_size)
             class ProcessingHeader(Header):
                 #     size = None
                 dtype = None
                 blockSize = None
                 profilesPerBlock = None
                 dataBlocksPerFile = None
                 nWindows = None
                 processFlags = None
                 nCohInt = None
                 nIncohInt = None
                 totalSpectra = None
                 structure = PROCESSING_STRUCTURE
                 flag_dc = None
                 flag_cspc = None
                 def __init__(self, dtype=0, blockSize=0, profilesPerBlock=0, dataBlocksPerFile=0, nWindows=0, processFlags=0, nCohInt=0,
                              nIncohInt=0, totalSpectra=0, nHeights=0, firstHeight=0, deltaHeight=0, samplesWin=0, spectraComb=0, nCode=0,
                              code=0, nBaud=None, shif_fft=False, flag_dc=False, flag_cspc=False, flag_decode=False, flag_deflip=False
                              ):
                     #         self.size = 0
                     self.dtype = dtype
                     self.blockSize = blockSize
                     self.profilesPerBlock = 0
                     self.dataBlocksPerFile = 0
                     self.nWindows = 0
                     self.processFlags = 0
                     self.nCohInt = 0
                     self.nIncohInt = 0
                     self.totalSpectra = 0
                     self.nHeights = 0
                     self.firstHeight = 0
                     self.deltaHeight = 0
                     self.samplesWin = 0
                     self.spectraComb = 0
                     self.nCode = None
                     self.code = None
                     self.nBaud = None
                     self.shif_fft = False
                     self.flag_dc = False
                     self.flag_cspc = False
                     self.flag_decode = False
                     self.flag_deflip = False
                     self.length = 0
                 def read(self, fp):
                     self.length = 0
                     try:
                         startFp = fp.tell()
                     except Exception as e:
                         startFp = None
                         pass
                     try:
                         if hasattr(fp, 'read'):
                             header = numpy.fromfile(fp, PROCESSING_STRUCTURE, 1)
                         else:
                             header = numpy.fromstring(fp, PROCESSING_STRUCTURE, 1)
                         self.length += header.nbytes
                     except Exception as e:
                         print("ProcessingHeader: " + str(e))
                         return 0
                     size = int(header['nSize'][0])
                     self.dtype = int(header['nDataType'][0])
                     self.blockSize = int(header['nSizeOfDataBlock'][0])
                     self.profilesPerBlock = int(header['nProfilesperBlock'][0])
                     self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0])
                     self.nWindows = int(header['nNumWindows'][0])
                     self.processFlags = header['nProcessFlags']
                     self.nCohInt = int(header['nCoherentIntegrations'][0])
                     self.nIncohInt = int(header['nIncoherentIntegrations'][0])
                     self.totalSpectra = int(header['nTotalSpectra'][0])
                     try:
                         if hasattr(fp, 'read'):
                             samplingWindow = numpy.fromfile(
                                 fp, SAMPLING_STRUCTURE, self.nWindows)
                         else:
                             samplingWindow = numpy.fromstring(
                                 fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
                         self.length += samplingWindow.nbytes
                     except Exception as e:
                         print("ProcessingHeader: " + str(e))
                         return 0
                     self.nHeights = int(numpy.sum(samplingWindow['nsa']))
                     self.firstHeight = float(samplingWindow['h0'][0])
                     self.deltaHeight = float(samplingWindow['dh'][0])
                     self.samplesWin = samplingWindow['nsa'][0]
                     try:
                         if hasattr(fp, 'read'):
                             self.spectraComb = numpy.fromfile(
                                 fp, 'u1', 2 * self.totalSpectra)
                         else:
                             self.spectraComb = numpy.fromstring(
                                 fp[self.length:], 'u1', 2 * self.totalSpectra)
                         self.length += self.spectraComb.nbytes
                     except Exception as e:
                         print("ProcessingHeader: " + str(e))
                         return 0
                     if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
                         self.nCode = int(numpy.fromfile(fp, '<u4', 1))
                         self.nBaud = int(numpy.fromfile(fp, '<u4', 1))
                         self.code = numpy.fromfile(
                             fp, '<f4', self.nCode * self.nBaud).reshape(self.nCode, self.nBaud)
                     if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP):
                         exp_name_len = int(numpy.fromfile(fp, '<u4', 1))
                         exp_name = numpy.fromfile(fp, 'u1', exp_name_len + 1)
                     if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
                         self.shif_fft = True
                     else:
                         self.shif_fft = False
                     if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
                         self.flag_dc = True
                     else:
                         self.flag_dc = False
                     if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA):
                         self.flag_decode = True
                     else:
                         self.flag_decode = False
                     if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA):
                         self.flag_deflip = True
                     else:
                         self.flag_deflip = False
                     nChannels = 0
                     nPairs = 0
                     pairList = []
                     for i in range(0, self.totalSpectra * 2, 2):
                         if self.spectraComb[i] == self.spectraComb[i + 1]:
                             nChannels = nChannels + 1  # par de canales iguales
                         else:
                             nPairs = nPairs + 1  # par de canales diferentes
                             pairList.append((self.spectraComb[i], self.spectraComb[i + 1]))
                     self.flag_cspc = False
                     if nPairs > 0:
                         self.flag_cspc = True
                     if startFp is not None:
                         endFp = size + startFp
                         if fp.tell() > endFp:
                             sys.stderr.write(
                                 "Warning: Processing header size is lower than it has to be")
                             return 0
                         if fp.tell() < endFp:
                             sys.stderr.write(
                                 "Warning: Processing header size is greater than it is considered")
                     return 1
                 def write(self, fp):
                     # Clear DEFINE_PROCESS_CODE
                     self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE)
                     headerTuple = (self.size,
                                    self.dtype,
                                    self.blockSize,
                                    self.profilesPerBlock,
                                    self.dataBlocksPerFile,
                                    self.nWindows,
                                    self.processFlags,
                                    self.nCohInt,
                                    self.nIncohInt,
                                    self.totalSpectra)
                     header = numpy.array(headerTuple, PROCESSING_STRUCTURE)
                     header.tofile(fp)
                     if self.nWindows != 0:
                         sampleWindowTuple = (
                             self.firstHeight, self.deltaHeight, self.samplesWin)
                         samplingWindow = numpy.array(sampleWindowTuple, SAMPLING_STRUCTURE)
                         samplingWindow.tofile(fp)
                     if self.totalSpectra != 0:
                         #             spectraComb = numpy.array([],numpy.dtype('u1'))
                         spectraComb = self.spectraComb
                         spectraComb.tofile(fp)
             #         if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
             #             nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba
             #             nCode.tofile(fp)
             #
             #             nBaud = numpy.array([self.nBaud], numpy.dtype('u4'))
             #             nBaud.tofile(fp)
             #
             #             code = self.code.reshape(self.nCode*self.nBaud)
             #             code = code.astype(numpy.dtype('<f4'))
             #             code.tofile(fp)
                     return 1
                 def get_size(self):
                     self.__size = 40 + 12 * self.nWindows + 2 * self.totalSpectra
             #         if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
             #             self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
             #             self.__size += 4 + 4 + 4 * self.nCode * self.nBaud
                     return self.__size
                 def set_size(self, value):
                     raise IOError("size is a property and it cannot be set, just read")
                     return
                 size = property(get_size, set_size)
             class RCfunction:
                 NONE = 0
                 FLIP = 1
                 CODE = 2
                 SAMPLING = 3
                 LIN6DIV256 = 4
                 SYNCHRO = 5
             class nCodeType:
                 NONE = 0
                 USERDEFINE = 1
                 BARKER2 = 2
                 BARKER3 = 3
                 BARKER4 = 4
                 BARKER5 = 5
                 BARKER7 = 6
                 BARKER11 = 7
                 BARKER13 = 8
                 AC128 = 9
                 COMPLEMENTARYCODE2 = 10
                 COMPLEMENTARYCODE4 = 11
                 COMPLEMENTARYCODE8 = 12
                 COMPLEMENTARYCODE16 = 13
                 COMPLEMENTARYCODE32 = 14
                 COMPLEMENTARYCODE64 = 15
                 COMPLEMENTARYCODE128 = 16
                 CODE_BINARY28 = 17
             class PROCFLAG:
                 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
                 DECODE_DATA = numpy.uint32(0x00000002)
                 SPECTRA_CALC = numpy.uint32(0x00000004)
                 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
                 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
                 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
                 DATATYPE_CHAR = numpy.uint32(0x00000040)
                 DATATYPE_SHORT = numpy.uint32(0x00000080)
                 DATATYPE_LONG = numpy.uint32(0x00000100)
                 DATATYPE_INT64 = numpy.uint32(0x00000200)
                 DATATYPE_FLOAT = numpy.uint32(0x00000400)
                 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
                 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
                 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
                 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
                 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
                 DEFLIP_DATA = numpy.uint32(0x00010000)
                 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
                 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
                 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
                 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
                 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
                 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
                 EXP_NAME_ESP = numpy.uint32(0x00200000)
                 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
                 OPERATION_MASK = numpy.uint32(0x0000003F)
                 DATATYPE_MASK = numpy.uint32(0x00000FC0)
                 DATAARRANGE_MASK = numpy.uint32(0x00007000)
                 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
             dtype0 = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
             dtype1 = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
             dtype2 = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
             dtype3 = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
             dtype4 = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
             dtype5 = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
             NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
             PROCFLAG_DTYPE_LIST = [PROCFLAG.DATATYPE_CHAR,
                                    PROCFLAG.DATATYPE_SHORT,
                                    PROCFLAG.DATATYPE_LONG,
                                    PROCFLAG.DATATYPE_INT64,
                                    PROCFLAG.DATATYPE_FLOAT,
                                    PROCFLAG.DATATYPE_DOUBLE]
             DTYPE_WIDTH = [1, 2, 4, 8, 4, 8]
             def get_dtype_index(numpy_dtype):
                 index = None
                 for i in range(len(NUMPY_DTYPE_LIST)):
                     if numpy_dtype == NUMPY_DTYPE_LIST[i]:
                         index = i
                         break
                 return index
             def get_numpy_dtype(index):
                 return NUMPY_DTYPE_LIST[index]
             def get_procflag_dtype(index):
                 return PROCFLAG_DTYPE_LIST[index]
             def get_dtype_width(index):
-                return DTYPE_WIDTH[index]
  No newline at end of file
+                return DTYPE_WIDTH[index]

schainpy/model/graphics/jroplot_base.py +23 -25

             import os
             import sys
             import zmq
             import time
             import numpy
             import datetime
             from functools import wraps
             from threading import Thread
             import matplotlib
             if 'BACKEND' in os.environ:
                 matplotlib.use(os.environ['BACKEND'])
             elif 'linux' in sys.platform:
                 matplotlib.use("TkAgg")
             elif 'darwin' in sys.platform:
                 matplotlib.use('WxAgg')
             else:
                 from schainpy.utils import log
                 log.warning('Using default Backend="Agg"', 'INFO')
                 matplotlib.use('Agg')
             import matplotlib.pyplot as plt
             from matplotlib.patches import Polygon
             from mpl_toolkits.axes_grid1 import make_axes_locatable
             from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
             from schainpy.model.data.jrodata import PlotterData
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.utils import log
             jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
             blu_values = matplotlib.pyplot.get_cmap(
                 'seismic_r', 20)(numpy.arange(20))[10:15]
             ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
                 'jro', numpy.vstack((blu_values, jet_values)))
             matplotlib.pyplot.register_cmap(cmap=ncmap)
             CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis',
                                                'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
             EARTH_RADIUS = 6.3710e3
             def ll2xy(lat1, lon1, lat2, lon2):
                 p = 0.017453292519943295
                 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
                     numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
                 r = 12742 * numpy.arcsin(numpy.sqrt(a))
                 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
                                       * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
                 theta = -theta + numpy.pi/2
                 return r*numpy.cos(theta), r*numpy.sin(theta)
             def km2deg(km):
                 '''
                 Convert distance in km to degrees
                 '''
                 return numpy.rad2deg(km/EARTH_RADIUS)
             def figpause(interval):
                 backend = plt.rcParams['backend']
                 if backend in matplotlib.rcsetup.interactive_bk:
                     figManager = matplotlib._pylab_helpers.Gcf.get_active()
                     if figManager is not None:
                         canvas = figManager.canvas
                         if canvas.figure.stale:
                             canvas.draw()
                         try:
                             canvas.start_event_loop(interval)
                         except:
                             pass
                         return
             def popup(message):
                 '''
                 '''
                 fig = plt.figure(figsize=(12, 8), facecolor='r')
                 text = '\n'.join([s.strip() for s in message.split(':')])
                 fig.text(0.01, 0.5, text, ha='left', va='center',
                          size='20', weight='heavy', color='w')
                 fig.show()
                 figpause(1000)
             class Throttle(object):
                 '''
                 Decorator that prevents a function from being called more than once every
                 time period.
                 To create a function that cannot be called more than once a minute, but
                 will sleep until it can be called:
                 @Throttle(minutes=1)
                 def foo():
                   pass
                 for i in range(10):
                   foo()
                   print "This function has run %s times." % i
                 '''
                 def __init__(self, seconds=0, minutes=0, hours=0):
                     self.throttle_period = datetime.timedelta(
                         seconds=seconds, minutes=minutes, hours=hours
                     )
                     self.time_of_last_call = datetime.datetime.min
                 def __call__(self, fn):
                     @wraps(fn)
                     def wrapper(*args, **kwargs):
                         coerce = kwargs.pop('coerce', None)
                         if coerce:
                             self.time_of_last_call = datetime.datetime.now()
                             return fn(*args, **kwargs)
                         else:
                             now = datetime.datetime.now()
                             time_since_last_call = now - self.time_of_last_call
                             time_left = self.throttle_period - time_since_last_call
                             if time_left > datetime.timedelta(seconds=0):
                                 return
                         self.time_of_last_call = datetime.datetime.now()
                         return fn(*args, **kwargs)
                     return wrapper
             def apply_throttle(value):
                 @Throttle(seconds=value)
                 def fnThrottled(fn):
                     fn()
                 return fnThrottled
             @MPDecorator
             class Plot(Operation):
                 '''
                 Base class for Schain plotting operations
                 '''
                 CODE = 'Figure'
                 colormap = 'jet'
                 bgcolor = 'white'
                 __missing = 1E30
                 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
                              'zlimits', 'xlabel', 'ylabel', 'xaxis', 'cb_label', 'title',
                              'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',
                              'showprofile', 'decimation', 'pause']
                 def __init__(self):
                     Operation.__init__(self)
                     self.isConfig = False
                     self.isPlotConfig = False
                     self.save_counter = 1
                     self.sender_counter = 1
                     self.data = None
                 def __fmtTime(self, x, pos):
                     '''
                     '''
                     return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
                 def __setup(self, **kwargs):
                     '''
                     Initialize variables
                     '''
                     self.figures = []
                     self.axes = []
                     self.cb_axes = []
                     self.localtime = kwargs.pop('localtime', True)
                     self.show = kwargs.get('show', True)
                     self.save = kwargs.get('save', False)
                     self.save_period = kwargs.get('save_period', 1)
                     self.ftp = kwargs.get('ftp', False)
                     self.colormap = kwargs.get('colormap', self.colormap)
                     self.colormap_coh = kwargs.get('colormap_coh', 'jet')
                     self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
                     self.colormaps = kwargs.get('colormaps', None)
                     self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
                     self.showprofile = kwargs.get('showprofile', False)
                     self.title = kwargs.get('wintitle', self.CODE.upper())
                     self.cb_label = kwargs.get('cb_label', None)
                     self.cb_labels = kwargs.get('cb_labels', None)
                     self.labels = kwargs.get('labels', None)
                     self.xaxis = kwargs.get('xaxis', 'frequency')
                     self.zmin = kwargs.get('zmin', None)
                     self.zmax = kwargs.get('zmax', None)
                     self.zlimits = kwargs.get('zlimits', None)
                     self.xmin = kwargs.get('xmin', None)
                     self.xmax = kwargs.get('xmax', None)
                     self.xrange = kwargs.get('xrange', 24)
                     self.xscale = kwargs.get('xscale', None)
                     self.ymin = kwargs.get('ymin', None)
                     self.ymax = kwargs.get('ymax', None)
                     self.yscale = kwargs.get('yscale', None)
                     self.xlabel = kwargs.get('xlabel', None)
                     self.decimation = kwargs.get('decimation', None)
                     self.showSNR = kwargs.get('showSNR', False)
                     self.oneFigure = kwargs.get('oneFigure', True)
                     self.width = kwargs.get('width', None)
                     self.height = kwargs.get('height', None)
                     self.colorbar = kwargs.get('colorbar', True)
                     self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
                     self.channels = kwargs.get('channels', None)
                     self.titles = kwargs.get('titles', [])
                     self.polar = False
                     self.type = kwargs.get('type', 'iq')
                     self.grid = kwargs.get('grid', False)
                     self.pause = kwargs.get('pause', False)
                     self.save_labels = kwargs.get('save_labels', None)
                     self.realtime = kwargs.get('realtime', True)
                     self.buffering = kwargs.get('buffering', True)
                     self.throttle = kwargs.get('throttle', 2)
                     self.exp_code = kwargs.get('exp_code', None)
                     self.plot_server = kwargs.get('plot_server', False)
                     self.sender_period = kwargs.get('sender_period', 1)
                     self.__throttle_plot = apply_throttle(self.throttle)
                     self.data = PlotterData(
                         self.CODE, self.throttle, self.exp_code, self.buffering, snr=self.showSNR)
                     if self.plot_server:
                         if not self.plot_server.startswith('tcp://'):
                             self.plot_server = 'tcp://{}'.format(self.plot_server)
                         log.success(
                             'Sending to server: {}'.format(self.plot_server),
                             self.name
                         )
                     if 'plot_name' in kwargs:
                         self.plot_name = kwargs['plot_name']
                 def __setup_plot(self):
                     '''
                     Common setup for all figures, here figures and axes are created
                     '''
                     self.setup()
                     self.time_label = 'LT' if self.localtime else 'UTC'
                     if self.width is None:
                         self.width = 8
                     self.figures = []
                     self.axes = []
                     self.cb_axes = []
                     self.pf_axes = []
                     self.cmaps = []
                     size = '15%' if self.ncols == 1 else '30%'
                     pad = '4%' if self.ncols == 1 else '8%'
                     if self.oneFigure:
                         if self.height is None:
                             self.height = 1.4 * self.nrows + 1
                         fig = plt.figure(figsize=(self.width, self.height),
                                          edgecolor='k',
                                          facecolor='w')
                         self.figures.append(fig)
                         for n in range(self.nplots):
                             ax = fig.add_subplot(self.nrows, self.ncols,
                                                  n + 1, polar=self.polar)
                             ax.tick_params(labelsize=8)
                             ax.firsttime = True
                             ax.index = 0
                             ax.press = None
                             self.axes.append(ax)
                             if self.showprofile:
                                 cax = self.__add_axes(ax, size=size, pad=pad)
                                 cax.tick_params(labelsize=8)
                                 self.pf_axes.append(cax)
                     else:
                         if self.height is None:
                             self.height = 3
                         for n in range(self.nplots):
                             fig = plt.figure(figsize=(self.width, self.height),
                                              edgecolor='k',
                                              facecolor='w')
                             ax = fig.add_subplot(1, 1, 1, polar=self.polar)
                             ax.tick_params(labelsize=8)
                             ax.firsttime = True
                             ax.index = 0
                             ax.press = None
                             self.figures.append(fig)
                             self.axes.append(ax)
                             if self.showprofile:
                                 cax = self.__add_axes(ax, size=size, pad=pad)
                                 cax.tick_params(labelsize=8)
                                 self.pf_axes.append(cax)
                     for n in range(self.nrows):
                         if self.colormaps is not None:
                             cmap = plt.get_cmap(self.colormaps[n])
                         else:
                             cmap = plt.get_cmap(self.colormap)
                         cmap.set_bad(self.bgcolor, 1.)
                         self.cmaps.append(cmap)
                     for fig in self.figures:
                         fig.canvas.mpl_connect('key_press_event', self.OnKeyPress)
                         fig.canvas.mpl_connect('scroll_event', self.OnBtnScroll)
                         fig.canvas.mpl_connect('button_press_event', self.onBtnPress)
                         fig.canvas.mpl_connect('motion_notify_event', self.onMotion)
                         fig.canvas.mpl_connect('button_release_event', self.onBtnRelease)
                 def OnKeyPress(self, event):
                     '''
                     Event for pressing keys (up, down) change colormap
                     '''
                     ax = event.inaxes
                     if ax in self.axes:
                         if event.key == 'down':
                             ax.index += 1
                         elif event.key == 'up':
                             ax.index -= 1
                         if ax.index < 0:
                             ax.index = len(CMAPS) - 1
                         elif ax.index == len(CMAPS):
                             ax.index = 0
                         cmap = CMAPS[ax.index]
                         ax.cbar.set_cmap(cmap)
                         ax.cbar.draw_all()
                         ax.plt.set_cmap(cmap)
                         ax.cbar.patch.figure.canvas.draw()
                         self.colormap = cmap.name
                 def OnBtnScroll(self, event):
                     '''
                     Event for scrolling, scale figure
                     '''
                     cb_ax = event.inaxes
                     if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
                         ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
                         pt = ax.cbar.ax.bbox.get_points()[:, 1]
                         nrm = ax.cbar.norm
                         vmin, vmax, p0, p1, pS = (
                             nrm.vmin, nrm.vmax, pt[0], pt[1], event.y)
                         scale = 2 if event.step == 1 else 0.5
                         point = vmin + (vmax - vmin) / (p1 - p0) * (pS - p0)
                         ax.cbar.norm.vmin = point - scale * (point - vmin)
                         ax.cbar.norm.vmax = point - scale * (point - vmax)
                         ax.plt.set_norm(ax.cbar.norm)
                         ax.cbar.draw_all()
                         ax.cbar.patch.figure.canvas.draw()
                 def onBtnPress(self, event):
                     '''
                     Event for mouse button press
                     '''
                     cb_ax = event.inaxes
                     if cb_ax is None:
                         return
                     if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
                         cb_ax.press = event.x, event.y
                     else:
                         cb_ax.press = None
                 def onMotion(self, event):
                     '''
                     Event for move inside colorbar
                     '''
                     cb_ax = event.inaxes
                     if cb_ax is None:
                         return
                     if cb_ax not in [ax.cbar.ax for ax in self.axes if ax.cbar]:
                         return
                     if cb_ax.press is None:
                         return
                     ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
                     xprev, yprev = cb_ax.press
                     dx = event.x - xprev
                     dy = event.y - yprev
                     cb_ax.press = event.x, event.y
                     scale = ax.cbar.norm.vmax - ax.cbar.norm.vmin
                     perc = 0.03
                     if event.button == 1:
                         ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
                         ax.cbar.norm.vmax -= (perc * scale) * numpy.sign(dy)
                     elif event.button == 3:
                         ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
                         ax.cbar.norm.vmax += (perc * scale) * numpy.sign(dy)
                     ax.cbar.draw_all()
                     ax.plt.set_norm(ax.cbar.norm)
                     ax.cbar.patch.figure.canvas.draw()
                 def onBtnRelease(self, event):
                     '''
                     Event for mouse button release
                     '''
                     cb_ax = event.inaxes
                     if cb_ax is not None:
                         cb_ax.press = None
                 def __add_axes(self, ax, size='30%', pad='8%'):
                     '''
                     Add new axes to the given figure
                     '''
                     divider = make_axes_locatable(ax)
                     nax = divider.new_horizontal(size=size, pad=pad)
                     ax.figure.add_axes(nax)
                     return nax
                 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
                     '''
                     Create a masked array for missing data
                     '''
                     if x_buffer.shape[0] < 2:
                         return x_buffer, y_buffer, z_buffer
                     deltas = x_buffer[1:] - x_buffer[0:-1]
                     x_median = numpy.median(deltas)
                     index = numpy.where(deltas > 5 * x_median)
                     if len(index[0]) != 0:
                         z_buffer[::, index[0], ::] = self.__missing
                         z_buffer = numpy.ma.masked_inside(z_buffer,
 .99 * self.__missing,
 .01 * self.__missing)
                     return x_buffer, y_buffer, z_buffer
                 def decimate(self):
                     # dx = int(len(self.x)/self.__MAXNUMX) + 1
                     dy = int(len(self.y) / self.decimation) + 1
                     # x = self.x[::dx]
                     x = self.x
                     y = self.y[::dy]
                     z = self.z[::, ::, ::dy]
                     return x, y, z
                 def format(self):
                     '''
                     Set min and max values, labels, ticks and titles
                     '''
                     if self.xmin is None:
                         xmin = self.data.min_time
                     else:
                         if self.xaxis is 'time':
                             dt = self.getDateTime(self.data.min_time)
                             xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) -
                                     datetime.datetime(1970, 1, 1)).total_seconds()
                             if self.data.localtime:
                                 xmin += time.timezone
                         else:
                             xmin = self.xmin
                     if self.xmax is None:
                         xmax = xmin + self.xrange * 60 * 60
                     else:
                         if self.xaxis is 'time':
                             dt = self.getDateTime(self.data.max_time)
                             xmax = (dt.replace(hour=int(self.xmax), minute=59, second=59) -
                                     datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=1)).total_seconds()
                             if self.data.localtime:
                                 xmax += time.timezone
                         else:
                             xmax = self.xmax
                     ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
                     ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
                     #Y = numpy.array([1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000])
                     #i = 1 if numpy.where(
                     #    abs(ymax-ymin) <= Y)[0][0] < 0 else numpy.where(abs(ymax-ymin) <= Y)[0][0]
                     #ystep = Y[i] / 10.
                     dig = int(numpy.log10(ymax))
                     if dig == 0:
                         digD = len(str(ymax)) - 2
                         ydec = ymax*(10**digD)
                         dig = int(numpy.log10(ydec))
                         ystep = ((ydec + (10**(dig)))//10**(dig))*(10**(dig))
                         ystep = ystep/5
                         ystep = ystep/(10**digD)
                     else:
                         ystep = ((ymax + (10**(dig)))//10**(dig))*(10**(dig))
                         ystep = ystep/5
                     if self.xaxis is not 'time':
                         dig = int(numpy.log10(xmax))
                         if dig <= 0:
                             digD = len(str(xmax)) - 2
                             xdec = xmax*(10**digD)
                             dig = int(numpy.log10(xdec))
                             xstep = ((xdec + (10**(dig)))//10**(dig))*(10**(dig))
                             xstep = xstep*0.5
                             xstep = xstep/(10**digD)
                         else:
                             xstep = ((xmax + (10**(dig)))//10**(dig))*(10**(dig))
                             xstep = xstep/5
                     for n, ax in enumerate(self.axes):
                         if ax.firsttime:
                             ax.set_facecolor(self.bgcolor)
                             ax.yaxis.set_major_locator(MultipleLocator(ystep))
                             if self.xscale:
                                 ax.xaxis.set_major_formatter(FuncFormatter(
                                     lambda x, pos: '{0:g}'.format(x*self.xscale)))
                             if self.xscale:
                                 ax.yaxis.set_major_formatter(FuncFormatter(
                                     lambda x, pos: '{0:g}'.format(x*self.yscale)))
                             if self.xaxis is 'time':
                                 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
                                 ax.xaxis.set_major_locator(LinearLocator(9))
                             else:
                                 ax.xaxis.set_major_locator(MultipleLocator(xstep))
                             if self.xlabel is not None:
                                 ax.set_xlabel(self.xlabel)
                             ax.set_ylabel(self.ylabel)
                             ax.firsttime = False
                             if self.showprofile:
                                 self.pf_axes[n].set_ylim(ymin, ymax)
                                 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
                                 self.pf_axes[n].set_xlabel('dB')
                                 self.pf_axes[n].grid(b=True, axis='x')
                                 [tick.set_visible(False)
                                  for tick in self.pf_axes[n].get_yticklabels()]
                             if self.colorbar:
                                 ax.cbar = plt.colorbar(
                                     ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
                                 ax.cbar.ax.tick_params(labelsize=8)
                                 ax.cbar.ax.press = None
                                 if self.cb_label:
                                     ax.cbar.set_label(self.cb_label, size=8)
                                 elif self.cb_labels:
                                     ax.cbar.set_label(self.cb_labels[n], size=8)
                             else:
                                 ax.cbar = None
                         if self.grid:
                             ax.grid(True)
                         if not self.polar:
                             ax.set_xlim(xmin, xmax)
                             ax.set_ylim(ymin, ymax)
                             ax.set_title('{} {} {}'.format(
                                 self.titles[n],
                                 self.getDateTime(self.data.max_time).strftime(
                                     '%Y-%m-%d %H:%M:%S'),
                                 self.time_label),
                                 size=8)
                         else:
                             ax.set_title('{}'.format(self.titles[n]), size=8)
                             ax.set_ylim(0, 90)
                             ax.set_yticks(numpy.arange(0, 90, 20))
                             ax.yaxis.labelpad = 40
                 def clear_figures(self):
                     '''
                     Reset axes for redraw plots
                     '''
                     for ax in self.axes:
                         ax.clear()
                         ax.firsttime = True
                         if ax.cbar:
                             ax.cbar.remove()
                 def __plot(self):
                     '''
                     Main function to plot, format and save figures
                     '''
                     try:
                         self.plot()
                         self.format()
                     except Exception as e:
                        log.warning('{} Plot could not be updated... check data'.format(
                            self.CODE), self.name)
                        log.error(str(e), '')
                        return
                     for n, fig in enumerate(self.figures):
                         if self.nrows == 0 or self.nplots == 0:
                             log.warning('No data', self.name)
                             fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
                             fig.canvas.manager.set_window_title(self.CODE)
                             continue
                         fig.tight_layout()
                         fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
                                                                              self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
                         fig.canvas.draw()
                         if self.show:
                             fig.show()
                             figpause(0.1)
                         if self.save:
                             self.save_figure(n)
                     if self.plot_server:
                         self.send_to_server()
                         # t = Thread(target=self.send_to_server)
                         # t.start()
                 def save_figure(self, n):
                     '''
                     '''
                     if self.save_counter < self.save_period:
                         self.save_counter += 1
                         return
                     self.save_counter = 1
                     fig = self.figures[n]
                     if self.save_labels:
                         labels = self.save_labels
                     else:
                         labels = list(range(self.nrows))
                     if self.oneFigure:
                         label = ''
                     else:
                         label = '-{}'.format(labels[n])
                     figname = os.path.join(
                         self.save,
                         self.CODE,
                         '{}{}_{}.png'.format(
                             self.CODE,
                             label,
-                            self.getDateTime(self.data.max_time).strftime(
+                            self.getDateTime(self.data.max_time).strftime('%Y%m%d_%H%M%S'),
-                                '%Y%m%d_%H%M%S'
-                                ),
                             )
                         )
                     log.log('Saving figure: {}'.format(figname), self.name)
                     if not os.path.isdir(os.path.dirname(figname)):
                         os.makedirs(os.path.dirname(figname))
                     fig.savefig(figname)
                     if self.realtime:
                         figname = os.path.join(
                             self.save,
                             '{}{}_{}.png'.format(
                                 self.CODE,
                                 label,
                                 self.getDateTime(self.data.min_time).strftime(
                                     '%Y%m%d'
                                     ),
                                 )
                             )
                         fig.savefig(figname)
                 def send_to_server(self):
                     '''
                     '''
                     if self.sender_counter < self.sender_period:
                         self.sender_counter += 1
                         return
                     self.sender_counter = 1
                     self.data.meta['titles'] = self.titles
                     retries = 2
                     while True:
                         self.socket.send_string(self.data.jsonify(self.plot_name, self.plot_type))
                         socks = dict(self.poll.poll(5000))
                         if socks.get(self.socket) == zmq.POLLIN:
                             reply = self.socket.recv_string()
                             if reply == 'ok':
                                 log.log("Response from server ok", self.name)
                                 break
                             else:
                                 log.warning(
                                     "Malformed reply from server: {}".format(reply), self.name)
                         else:
                             log.warning(
                                 "No response from server, retrying...", self.name)
                         self.socket.setsockopt(zmq.LINGER, 0)
                         self.socket.close()
                         self.poll.unregister(self.socket)
                         retries -= 1
                         if retries == 0:
                             log.error(
                                 "Server seems to be offline, abandoning", self.name)
                             self.socket = self.context.socket(zmq.REQ)
                             self.socket.connect(self.plot_server)
                             self.poll.register(self.socket, zmq.POLLIN)
                             time.sleep(1)
                             break
                         self.socket = self.context.socket(zmq.REQ)
                         self.socket.connect(self.plot_server)
                         self.poll.register(self.socket, zmq.POLLIN)
                         time.sleep(0.5)
                 def setup(self):
                     '''
                     This method should be implemented in the child class, the following
                     attributes should be set:
                     self.nrows: number of rows
                     self.ncols: number of cols
                     self.nplots: number of plots (channels or pairs)
                     self.ylabel: label for Y axes
                     self.titles: list of axes title
                     '''
                     raise NotImplementedError
                 def plot(self):
                     '''
                     Must be defined in the child class
                     '''
                     raise NotImplementedError
                 def run(self, dataOut, **kwargs):
                     '''
                     Main plotting routine
                     '''
                     if self.isConfig is False:
                         self.__setup(**kwargs)
                         if dataOut.type == 'Parameters':
                             t = dataOut.utctimeInit
                         else:
                             t = dataOut.utctime
                         if dataOut.useLocalTime:
                             self.getDateTime = datetime.datetime.fromtimestamp
                             if not self.localtime:
                                 t += time.timezone
                         else:
                             self.getDateTime = datetime.datetime.utcfromtimestamp
                             if self.localtime:
                                 t -= time.timezone
                         if 'buffer' in self.plot_type:
                             if self.xmin is None:
                                 self.tmin = t
                             else:
                                 self.tmin = (
                                     self.getDateTime(t).replace(
                                         hour=self.xmin,
                                         minute=0,
                                         second=0) - self.getDateTime(0)).total_seconds()
                         self.data.setup()
                         self.isConfig = True
                         if self.plot_server:
                             self.context = zmq.Context()
                             self.socket = self.context.socket(zmq.REQ)
                             self.socket.connect(self.plot_server)
                             self.poll = zmq.Poller()
                             self.poll.register(self.socket, zmq.POLLIN)
                     if dataOut.type == 'Parameters':
                         tm = dataOut.utctimeInit
                     else:
                         tm = dataOut.utctime
                     if not dataOut.useLocalTime and self.localtime:
                         tm -= time.timezone
                     if dataOut.useLocalTime and not self.localtime:
                         tm += time.timezone
                     if self.xaxis is 'time' and self.data and (tm - self.tmin) >= self.xrange*60*60:
                         self.save_counter = self.save_period
                         self.__plot()
                         self.xmin += self.xrange
                         if self.xmin >= 24:
                             self.xmin -= 24
                         self.tmin += self.xrange*60*60
                         self.data.setup()
                         self.clear_figures()
                     self.data.update(dataOut, tm)
                     if self.isPlotConfig is False:
                         self.__setup_plot()
                         self.isPlotConfig = True
                     if self.realtime:
                         self.__plot()
                     else:
                         self.__throttle_plot(self.__plot)#, coerce=coerce)
                 def close(self):
                     if self.data:
                         self.save_counter = self.save_period
                         self.__plot()
                     if self.data and self.pause:
                         figpause(10)

schainpy/model/io/jroIO_kamisr.py +205 -185

             '''
             Created on Set 9, 2015
             @author: roj-idl71 Karim Kuyeng
             '''
             import os
             import sys
             import glob
             import fnmatch
             import datetime
             import time
             import re
             import h5py
             import numpy
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
-            from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
+            from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from numpy import imag
+            @MPDecorator
             class AMISRReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self)
                     self.set = None
                     self.subset = None
                     self.extension_file = '.h5'
                     self.dtc_str = 'dtc'
                     self.dtc_id = 0
                     self.status = True
                     self.isConfig = False
                     self.dirnameList = []
                     self.filenameList = []
                     self.fileIndex = None
                     self.flagNoMoreFiles = False
                     self.flagIsNewFile = 0
                     self.filename = ''
                     self.amisrFilePointer = None
-                    self.dataset = None
+                    #self.dataset = None
                     self.profileIndex = 0
                     self.beamCodeByFrame = None
                     self.radacTimeByFrame = None
                     self.dataset = None
                     self.__firstFile = True
                     self.buffer = None
                     self.timezone = 'ut'
                     self.__waitForNewFile = 20
                     self.__filename_online = None
                     #Is really necessary create the output object in the initializer
                     self.dataOut = Voltage()
+                    self.dataOut.error=False
                 def setup(self,path=None,
                                 startDate=None,
                                 endDate=None,
                                 startTime=None,
                                 endTime=None,
                                 walk=True,
                                 timezone='ut',
                                 all=0,
                                 code = None,
                                 nCode = 0,
                                 nBaud = 0,
                                 online=False):
+                    #print ("T",path)
                     self.timezone = timezone
                     self.all = all
                     self.online = online
                     self.code = code
                     self.nCode = int(nCode)
                     self.nBaud = int(nBaud)
                     #self.findFiles()
                     if not(online):
                         #Busqueda de archivos offline
                         self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
                     else:
                         self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
                     if not(self.filenameList):
                         print("There is no files into the folder: %s"%(path))
                         sys.exit(-1)
                     self.fileIndex = -1
                     self.readNextFile(online)
                     '''
                     Add code
                     '''
                     self.isConfig = True
                     pass
                 def readAMISRHeader(self,fp):
                     header = 'Raw11/Data/RadacHeader'
                     self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
                     self.beamCode = fp.get('Raw11/Data/Beamcodes') # NUMBER OF CHANNELS AND IDENTIFY POSITION TO CREATE A FILE WITH THAT INFO
                     #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
                     self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
                     self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
                     self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
                     self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
                     self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
                     self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
                     self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
                     self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
                     self.frequency =  fp.get('Rx/Frequency')
                     txAus = fp.get('Raw11/Data/Pulsewidth')
                     self.nblocks = self.pulseCount.shape[0] #nblocks
                     self.nprofiles = self.pulseCount.shape[1] #nprofile
                     self.nsa = self.nsamplesPulse[0,0] #ngates
                     self.nchannels = self.beamCode.shape[1]
                     self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
                     #self.__waitForNewFile = self.nblocks  # wait depending on the number of blocks since each block is 1 sec
                     self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
                     #filling radar controller header parameters
                     self.__ippKm = self.ippSeconds *.15*1e6 # in km
                     self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
                     self.__txB = 0
                     nWindows=1
                     self.__nSamples = self.nsa
                     self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
                     self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
                     #for now until understand why the code saved is different (code included even though code not in tuf file)
                     #self.__codeType = 0
                    # self.__nCode = None
                    # self.__nBaud = None
                     self.__code = self.code
                     self.__codeType = 0
                     if self.code != None:
                         self.__codeType = 1
                     self.__nCode = self.nCode
                     self.__nBaud = self.nBaud
                     #self.__code = 0
                     #filling system header parameters
                     self.__nSamples = self.nsa
                     self.newProfiles = self.nprofiles/self.nchannels
                     self.__channelList = list(range(self.nchannels))
                     self.__frequency = self.frequency[0][0]
                 def createBuffers(self):
                     pass
                 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.walk = walk
                 def __checkPath(self):
                     if os.path.exists(self.path):
                         self.status = 1
                     else:
                         self.status = 0
                         print('Path:%s does not exists'%self.path)
                     return
                 def __selDates(self, amisr_dirname_format):
                     try:
                         year = int(amisr_dirname_format[0:4])
                         month = int(amisr_dirname_format[4:6])
                         dom = int(amisr_dirname_format[6:8])
                         thisDate = datetime.date(year,month,dom)
                         if (thisDate>=self.startDate and thisDate <= self.endDate):
                             return amisr_dirname_format
                     except:
                         return None
                 def __findDataForDates(self,online=False):
                     if not(self.status):
                         return None
                     pat = '\d+.\d+'
                     dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
                     dirnameList = [x for x in dirnameList if x!=None]
                     dirnameList = [x.string for x in dirnameList]
                     if not(online):
                         dirnameList = [self.__selDates(x) for x in dirnameList]
                         dirnameList = [x for x in dirnameList if x!=None]
                     if len(dirnameList)>0:
                         self.status = 1
                         self.dirnameList = dirnameList
                         self.dirnameList.sort()
                     else:
                         self.status = 0
                         return None
                 def __getTimeFromData(self):
                     startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
                     endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                     print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
                     print('........................................')
                     filter_filenameList = []
                     self.filenameList.sort()
                     #for i in range(len(self.filenameList)-1):
                     for i in range(len(self.filenameList)):
                         filename = self.filenameList[i]
                         fp = h5py.File(filename,'r')
                         time_str = fp.get('Time/RadacTimeString')
-                        startDateTimeStr_File = time_str[0][0].split('.')[0]
+                        startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
+                        #startDateTimeStr_File = "2019-12-16 09:21:11"
                         junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                         startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
-                        endDateTimeStr_File = time_str[-1][-1].split('.')[0]
+                        #endDateTimeStr_File = "2019-12-16 11:10:11"
+                        endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
                         junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                         endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                         fp.close()
+                        #print("check time", startDateTime_File)
                         if self.timezone == 'lt':
                             startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
                             endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
                         if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
                             #self.filenameList.remove(filename)
                             filter_filenameList.append(filename)
                         if (endDateTime_File>=endDateTime_Reader):
                             break
                     filter_filenameList.sort()
                     self.filenameList = filter_filenameList
                     return 1
                 def __filterByGlob1(self, dirName):
                     filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
                     filter_files.sort()
                     filterDict = {}
                     filterDict.setdefault(dirName)
                     filterDict[dirName] = filter_files
                     return filterDict
                 def __getFilenameList(self, fileListInKeys, dirList):
                     for value in fileListInKeys:
                         dirName = list(value.keys())[0]
                         for file in value[dirName]:
                             filename = os.path.join(dirName, file)
                             self.filenameList.append(filename)
                 def __selectDataForTimes(self, online=False):
                     #aun no esta implementado el filtro for tiempo
                     if not(self.status):
                         return None
                     dirList = [os.path.join(self.path,x) for x in self.dirnameList]
                     fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
                     self.__getFilenameList(fileListInKeys, dirList)
                     if not(online):
                         #filtro por tiempo
                         if not(self.all):
                             self.__getTimeFromData()
                         if len(self.filenameList)>0:
                             self.status = 1
                             self.filenameList.sort()
                         else:
                             self.status = 0
                             return None
                     else:
                         #get the last file - 1
                         self.filenameList = [self.filenameList[-2]]
                     new_dirnameList = []
                     for dirname in self.dirnameList:
                         junk = numpy.array([dirname in x for x in self.filenameList])
                         junk_sum = junk.sum()
                         if junk_sum > 0:
                             new_dirnameList.append(dirname)
                     self.dirnameList = new_dirnameList
                     return 1
                 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),walk=True):
                     if endDate ==None:
                      startDate = datetime.datetime.utcnow().date()
                      endDate = datetime.datetime.utcnow().date()
                     self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
                     self.__checkPath()
                     self.__findDataForDates(online=True)
                     self.dirnameList = [self.dirnameList[-1]]
                     self.__selectDataForTimes(online=True)
                     return
                 def searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         walk=True):
                     self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
                     self.__checkPath()
                     self.__findDataForDates()
                     self.__selectDataForTimes()
                     for i in range(len(self.filenameList)):
                         print("%s" %(self.filenameList[i]))
                     return
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print("No more Files")
+                            self.dataOut.error = True
                             return 0
                         filename = self.filenameList[idFile]
                         amisrFilePointer = h5py.File(filename,'r')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.amisrFilePointer = amisrFilePointer
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def __setNextFileOnline(self):
                     filename = self.filenameList[0]
                     if self.__filename_online != None:
                         self.__selectDataForTimes(online=True)
                         filename = self.filenameList[0]
                         wait = 0
                         while self.__filename_online == filename:
                             print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
                             if wait == 5:
                                 return 0
                             sleep(self.__waitForNewFile)
                             self.__selectDataForTimes(online=True)
                             filename = self.filenameList[0]
                             wait += 1
                     self.__filename_online = filename
                     self.amisrFilePointer = h5py.File(filename,'r')
                     self.flagIsNewFile = 1
                     self.filename = filename
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def readData(self):
                     buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
                     re = buffer[:,:,:,0]
                     im = buffer[:,:,:,1]
                     dataset = re + im*1j
                     self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
                     timeset = self.radacTime[:,0]
                     return dataset,timeset
                 def reshapeData(self):
                 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
                     channels = self.beamCodeByPulse[0,:]
                     nchan = self.nchannels
                     #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
                     nblocks = self.nblocks
                     nsamples = self.nsa
                     #Dimensions : nChannels, nProfiles, nSamples
-                    new_block = numpy.empty((nblocks, nchan, self.newProfiles, nsamples), dtype="complex64")
+                    new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
                     ############################################
                     for thisChannel in range(nchan):
                         new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[0][thisChannel])[0],:]
                     new_block = numpy.transpose(new_block, (1,0,2,3))
                     new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
                     return new_block
                 def updateIndexes(self):
                     pass
                 def fillJROHeader(self):
                     #fill radar controller header
-                    self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
+                    self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
                                                                                   txA=self.__txA,
                                                                                   txB=0,
                                                                                   nWindows=1,
                                                                                   nHeights=self.__nSamples,
                                                                                   firstHeight=self.__firstHeight,
                                                                                   deltaHeight=self.__deltaHeight,
                                                                                   codeType=self.__codeType,
                                                                                   nCode=self.__nCode, nBaud=self.__nBaud,
                                                                                   code = self.__code,
                                                                                   fClock=1)
                     #fill system header
                     self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                 nProfiles=self.newProfiles,
                                                                 nChannels=len(self.__channelList),
                                                                 adcResolution=14,
-                                                                pciDioBusWith=32)
+                                                                pciDioBusWidth=32)
                     self.dataOut.type = "Voltage"
                     self.dataOut.data = None
                     self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
             #        self.dataOut.nChannels = 0
             #        self.dataOut.nHeights = 0
                     self.dataOut.nProfiles = self.newProfiles*self.nblocks
                     #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
                     ranges = numpy.reshape(self.rangeFromFile.value,(-1))
                     self.dataOut.heightList =  ranges/1000.0 #km
                     self.dataOut.channelList = self.__channelList
                     self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
             #        self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData = True
                     #Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime = None
                     #self.dataOut.timeZone = -5 #self.__timezone/60  #timezone like jroheader, difference in minutes between UTC and localtime
                     if self.timezone == 'lt':
                         self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
                     else:
                         self.dataOut.timeZone = 0 #by default time is UTC
                     self.dataOut.dstFlag = 0
                     self.dataOut.errorCount = 0
                     self.dataOut.nCohInt = 1
                     self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
                     self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.ippSeconds = self.ippSeconds
                     #Time interval between profiles
                     #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.__frequency
                     self.dataOut.realtime = self.online
                     pass
                 def readNextFile(self,online=False):
                     if not(online):
                         newFile = self.__setNextFileOffline()
                     else:
                         newFile = self.__setNextFileOnline()
                     if not(newFile):
                         return 0
                     #if self.__firstFile:
                     self.readAMISRHeader(self.amisrFilePointer)
                     self.createBuffers()
                     self.fillJROHeader()
                     #self.__firstFile = False
                     self.dataset,self.timeset = self.readData()
                     if self.endDate!=None:
                      endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                      time_str = self.amisrFilePointer.get('Time/RadacTimeString')
-                     startDateTimeStr_File = time_str[0][0].split('.')[0]
+                     startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
                      junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                      startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                      if self.timezone == 'lt':
                       startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
                      if (startDateTime_File>endDateTime_Reader):
                          return 0
                     self.jrodataset = self.reshapeData()
                     #----self.updateIndexes()
                     self.profileIndex = 0
                     return 1
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= (self.newProfiles*self.nblocks):
                         return 1
                     return 0
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         return 0
                     if self.__hasNotDataInBuffer():
                         if not (self.readNextFile(self.online)):
                             return 0
-                    if self.dataset is None: # setear esta condicion cuando no hayan datos por leers
+                    if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
                         self.dataOut.flagNoData = True
                         return 0
                     #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
                     self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
+                    #print("R_t",self.timeset)
                     #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
                     #verificar basic header de jro data y ver si es compatible con este valor
                     #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
                     indexprof = numpy.mod(self.profileIndex, self.newProfiles)
                     indexblock = self.profileIndex/self.newProfiles
-                    #print indexblock, indexprof
+                    #print (indexblock, indexprof)
-                    self.dataOut.utctime = self.timeset[indexblock] + (indexprof * self.ippSeconds * self.nchannels)
+                    diffUTC = 1.8e4 #UTC diference from peru in seconds --Joab
+                    diffUTC = 0
+                    t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
+                    #cambio posible 18/02/2020
+                    #print("utc :",indexblock," __ ",t_comp)
+                    #print(numpy.shape(self.timeset))
+                    self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
+                    #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
+                    #print(self.dataOut.utctime)
                     self.dataOut.profileIndex = self.profileIndex
                     self.dataOut.flagNoData = False
             #         if indexprof == 0:
             #             print self.dataOut.utctime
                     self.profileIndex += 1
                     return self.dataOut.data
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()

schainpy/model/io/jroIO_param.py +36 -36

             import numpy
             import time
             import os
             import h5py
             import re
             import datetime
             import schainpy.admin
             from schainpy.model.data.jrodata import *
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.io.jroIO_base import *
             from schainpy.utils import log
             @MPDecorator
             class ParamReader(JRODataReader,ProcessingUnit):
                 '''
                 Reads HDF5 format files
                 path
                 startDate
                 endDate
                 startTime
                 endTime
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 timezone = None
                 startTime = None
                 endTime = None
                 fileIndex = None
                 utcList = None      #To select data in the utctime list
                 blockList = None    #List to blocks to be read from the file
                 blocksPerFile = None    #Number of blocks to be read
                 blockIndex = None
                 path = None
                 #List of Files
                 filenameList = None
                 datetimeList = None
                 #Hdf5 File
                 listMetaname = None
                 listMeta = None
                 listDataname = None
                 listData = None
                 listShapes = None
                 fp = None
                 #dataOut reconstruction
                 dataOut = None
                 def __init__(self):#, **kwargs):
                     ProcessingUnit.__init__(self) #, **kwargs)
                     self.dataOut = Parameters()
                     return
                 def setup(self, **kwargs):
                     path = kwargs['path']
                     startDate = kwargs['startDate']
                     endDate = kwargs['endDate']
                     startTime = kwargs['startTime']
                     endTime = kwargs['endTime']
                     walk = kwargs['walk']
                     if 'ext' in kwargs:
                         ext = kwargs['ext']
                     else:
                         ext = '.hdf5'
                     if 'timezone' in kwargs:
                         self.timezone = kwargs['timezone']
                     else:
                         self.timezone = 'lt'
                     print("[Reading] Searching files in offline mode ...")
                     pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            ext=ext, walk=walk)
                     if not(filenameList):
                         print("There is no files into the folder: %s"%(path))
                         sys.exit(-1)
                     self.fileIndex = -1
                     self.startTime = startTime
                     self.endTime = endTime
                     self.__readMetadata()
                     self.__setNextFileOffline()
                     return
                 def searchFilesOffLine(self,
                                         path,
                                         startDate=None,
                                         endDate=None,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         ext='.hdf5',
                                         walk=True):
                     expLabel = ''
                     self.filenameList = []
                     self.datetimeList = []
                     pathList = []
                     JRODataObj = JRODataReader()
                     dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
                     if dateList == []:
                         print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime()))
                         return None, None
                     if len(dateList) > 1:
                         print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
                     else:
                         print("[Reading] data was found for the date %s" %(dateList[0]))
                     filenameList = []
                     datetimeList = []
                     #----------------------------------------------------------------------------------
                     for thisPath in pathList:
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             if not isFileInDateRange(filename, startDate, endDate):
                                 continue
                             thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
                         return None, None
                     print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
                     print()
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
                     """
                     Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                     Inputs:
                         filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                         startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                         endDate            :    fecha final del rango seleccionado en formato datetime.date
                         startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                         endTime            :    tiempo final del rango seleccionado en formato datetime.time
                     Return:
                         Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                         fecha especificado, de lo contrario retorna False.
                     Excepciones:
                         Si el archivo no existe o no puede ser abierto
                         Si la cabecera no puede ser leida.
                     """
                     try:
                         fp = h5py.File(filename,'r')
                         grp1 = fp['Data']
                     except IOError:
                         traceback.print_exc()
                         raise IOError("The file %s can't be opened" %(filename))
                     #In case has utctime attribute
                     grp2 = grp1['utctime']
              #         thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
                     thisUtcTime = grp2.value[0]
                     fp.close()
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
                     #General case
                     #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                     #-----------o----------------------------o-----------
                     #       startTime                     endTime
                     if endTime >= startTime:
                         thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
                         if numpy.any(thisUtcLog):   #If there is one block between the hours mentioned
                             return thisDatetime
                         return None
                     #If endTime < startTime then endTime belongs to the next day
                     #<<<<<<<<<<<o                            o>>>>>>>>>>>
                     #-----------o----------------------------o-----------
                     #        endTime                    startTime
                     if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
                         return None
                     if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
                         return None
                     if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
                         return None
                     return thisDatetime
                 def __setNextFileOffline(self):
                     self.fileIndex += 1
                     idFile = self.fileIndex
                     if not(idFile < len(self.filenameList)):
                         raise schainpy.admin.SchainError("No more Files")
                         return 0
                     filename = self.filenameList[idFile]
                     filePointer = h5py.File(filename,'r')
                     self.filename = filename
                     self.fp = filePointer
                     print("Setting the file: %s"%self.filename)
                     self.__setBlockList()
                     self.__readData()
                     self.blockIndex = 0
                     return 1
                 def __setBlockList(self):
                     '''
                     Selects the data within the times defined
                     self.fp
                     self.startTime
                     self.endTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     fp = self.fp
                     startTime = self.startTime
                     endTime = self.endTime
                     grp = fp['Data']
                     thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
                     #ERROOOOR
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     return
                 def __readMetadata(self):
                     '''
                     Reads Metadata
                     self.pathMeta
                     self.listShapes
                     self.listMetaname
                     self.listMeta
                     '''
                     filename = self.filenameList[0]
                     fp = h5py.File(filename,'r')
                     gp = fp['Metadata']
                     listMetaname = []
                     listMetadata = []
                     for item in list(gp.items()):
                         name = item[0]
                         if name=='array dimensions':
                             table = gp[name][:]
                             listShapes = {}
                             for shapes in table:
                                 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
                         else:
                             data = gp[name].value
                             listMetaname.append(name)
                             listMetadata.append(data)
                     self.listShapes = listShapes
                     self.listMetaname = listMetaname
                     self.listMeta = listMetadata
                     fp.close()
                     return
                 def __readData(self):
                     grp = self.fp['Data']
                     listdataname = []
                     listdata = []
                     for item in list(grp.items()):
                         name = item[0]
                         listdataname.append(name)
                         array = self.__setDataArray(grp[name],self.listShapes[name])
                         listdata.append(array)
                     self.listDataname = listdataname
                     self.listData = listdata
                     return
                 def __setDataArray(self, dataset, shapes):
                     nDims = shapes[0]
                     nDim2 = shapes[1]      #Dimension 0
                     nDim1 = shapes[2]      #Dimension 1, number of Points or Parameters
                     nDim0 = shapes[3]      #Dimension 2, number of samples or ranges
                     mode = shapes[4]        #Mode of storing
                     blockList = self.blockList
                     blocksPerFile = self.blocksPerFile
                     #Depending on what mode the data was stored
                     if mode == 0:       #Divided in channels
                         arrayData = dataset.value.astype(numpy.float)[0][blockList]
                     if mode == 1:     #Divided in parameter
                         strds = 'table'
                         nDatas = nDim1
                         newShapes = (blocksPerFile,nDim2,nDim0)
                     elif mode==2:       #Concatenated in a table
                         strds = 'table0'
                         arrayData = dataset[strds].value
                         #Selecting part of the dataset
                         utctime = arrayData[:,0]
                         u, indices = numpy.unique(utctime, return_index=True)
                         if blockList.size != indices.size:
                             indMin = indices[blockList[0]]
                             if blockList[1] + 1 >= indices.size:
                                 arrayData = arrayData[indMin:,:]
                             else:
                                 indMax = indices[blockList[1] + 1]
                                 arrayData = arrayData[indMin:indMax,:]
                         return arrayData
                     #    One dimension
                     if nDims == 0:
                         arrayData = dataset.value.astype(numpy.float)[0][blockList]
                     #    Two dimensions
                     elif nDims == 2:
                         arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
                         newShapes = (blocksPerFile,nDim0)
                         nDatas = nDim1
                         for i in range(nDatas):
                             data = dataset[strds + str(i)].value
                             arrayData[:,i,:] = data[blockList,:]
                     #    Three dimensions
                     else:
                         arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
                         for i in range(nDatas):
                             data = dataset[strds + str(i)].value
                             for b in range(blockList.size):
                                 arrayData[b,:,i,:] = data[:,:,blockList[b]]
                     return arrayData
                 def __setDataOut(self):
                     listMeta = self.listMeta
                     listMetaname = self.listMetaname
                     listDataname = self.listDataname
                     listData = self.listData
                     listShapes = self.listShapes
                     blockIndex = self.blockIndex
              #         blockList = self.blockList
                     for i in range(len(listMeta)):
                         setattr(self.dataOut,listMetaname[i],listMeta[i])
                     for j in range(len(listData)):
                         nShapes = listShapes[listDataname[j]][0]
                         mode = listShapes[listDataname[j]][4]
                         if nShapes == 1:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
                         elif nShapes > 1:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
                         elif mode==0:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
                         #Mode Meteors
                         elif mode ==2:
                             selectedData = self.__selectDataMode2(listData[j], blockIndex)
                             setattr(self.dataOut, listDataname[j], selectedData)
                     return
                 def __selectDataMode2(self, data, blockIndex):
                     utctime = data[:,0]
                     aux, indices = numpy.unique(utctime, return_inverse=True)
                     selInd = numpy.where(indices == blockIndex)[0]
                     selData = data[selInd,:]
                     return selData
                 def getData(self):
                     if self.blockIndex==self.blocksPerFile:
                          if not( self.__setNextFileOffline() ):
                             self.dataOut.flagNoData = True
                             return 0
                     self.__setDataOut()
                     self.dataOut.flagNoData = False
                     self.blockIndex += 1
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
                     return
             @MPDecorator
             class ParamWriter(Operation):
                 '''
                 HDF5 Writer, stores parameters data in HDF5 format files
                 path:             path where the files will be stored
                 blocksPerFile:    number of blocks that will be saved in per HDF5 format file
                 mode:             selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
                 metadataList:     list of attributes that will be stored as metadata
                 dataList:         list of attributes that will be stores as data
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 metaoptchar = "M"
                 metaFile = None
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 grp = None
                 ds = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 dsList = None   #List of dictionaries with dataset properties
                 tableDim = None
                 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
                 currentDay = None
                 lastTime = None
                 setType = None
                 def __init__(self):
                     Operation.__init__(self)
                     return
                 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataList = dataList
                     self.dataOut = dataOut
                     self.mode = mode
                     if self.mode is not None:
                         self.mode = numpy.zeros(len(self.dataList)) + mode
                     else:
                         self.mode = numpy.ones(len(self.dataList))
                     self.setType = setType
                     arrayDim = numpy.zeros((len(self.dataList),5))
                     #Table dimensions
                     dtype0 = self.dtype
                     tableList = []
                     #Dictionary and list of tables
                     dsList = []
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         dataAux = getattr(self.dataOut, self.dataList[i])
                         dsDict['variable'] = self.dataList[i]
                         #---------------------    Conditionals    ------------------------
                         #There is no data
                         if dataAux is None:
                             return 0
                         if isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
                             dsDict['mode'] = 0
                             dsDict['nDim'] = 0
                             arrayDim[i,0] = 0
                             dsList.append(dsDict)
                         #Mode 2: meteors
                         elif self.mode[i] == 2:
                             dsDict['dsName'] = 'table0'
                             dsDict['mode'] = 2      # Mode meteors
                             dsDict['shape'] = dataAux.shape[-1]
                             dsDict['nDim'] = 0
                             dsDict['dsNumber'] = 1
                             arrayDim[i,3] = dataAux.shape[-1]
                             arrayDim[i,4] = self.mode[i]         #Mode the data was stored
                             dsList.append(dsDict)
                         #Mode 1
                         else:
                             arrayDim0 = dataAux.shape       #Data dimensions
                             arrayDim[i,0] = len(arrayDim0)  #Number of array dimensions
                             arrayDim[i,4] = self.mode[i]         #Mode the data was stored
                             strtable = 'table'
                             dsDict['mode'] = 1      # Mode parameters
                             # Three-dimension arrays
                             if len(arrayDim0) == 3:
                                 arrayDim[i,1:-1] = numpy.array(arrayDim0)
                                 nTables = int(arrayDim[i,2])
                                 dsDict['dsNumber'] = nTables
                                 dsDict['shape'] = arrayDim[i,2:4]
                                 dsDict['nDim'] = 3
                                 for j in range(nTables):
                                     dsDict = dsDict.copy()
                                     dsDict['dsName'] = strtable + str(j)
                                     dsList.append(dsDict)
                             # Two-dimension arrays
                             elif len(arrayDim0) == 2:
                                 arrayDim[i,2:-1] = numpy.array(arrayDim0)
                                 nTables = int(arrayDim[i,2])
                                 dsDict['dsNumber'] = nTables
                                 dsDict['shape'] = arrayDim[i,3]
                                 dsDict['nDim'] = 2
                                 for j in range(nTables):
                                     dsDict = dsDict.copy()
                                     dsDict['dsName'] = strtable + str(j)
                                     dsList.append(dsDict)
                             # One-dimension arrays
                             elif len(arrayDim0) == 1:
                                 arrayDim[i,3] = arrayDim0[0]
                                 dsDict['shape'] = arrayDim0[0]
                                 dsDict['dsNumber'] = 1
                                 dsDict['dsName'] = strtable + str(0)
                                 dsDict['nDim'] = 1
                                 dsList.append(dsDict)
                         table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
                         tableList.append(table)
                     self.dsList = dsList
                     self.tableDim = numpy.array(tableList, dtype = dtype0)
                     self.blockIndex = 0
                     timeTuple = time.localtime(dataOut.utctime)
                     self.currentDay = timeTuple.tm_yday
                 def putMetadata(self):
                     fp = self.createMetadataFile()
                     self.writeMetadata(fp)
                     fp.close()
                     return
                 def createMetadataFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = ''
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         filesList = sorted( filesList, key=str.lower )
                         if len( filesList ) > 0:
                             filesList = [k for k in filesList if k.startswith(self.metaoptchar)]
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     filename = os.path.join( path, subfolder, file )
                     self.metaFile = file
                     #Setting HDF5 File
                     fp = h5py.File(filename,'w')
                     return fp
                 def writeMetadata(self, fp):
                     grp = fp.create_group("Metadata")
                     grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
                     for i in range(len(self.metadataList)):
                         grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
                     return
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     if self.lastTime is None:
                         self.lastTime = currentTime
                     #Day
                     timeTuple = time.localtime(currentTime)
                     dataDay = timeTuple.tm_yday
                     #Time
                     timeDiff = currentTime - self.lastTime
                     #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                     if dataDay != self.currentDay:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > 3*60*60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     mode = self.mode
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if os.path.exists(fullpath):
                         filesList = os.listdir( fullpath )
-                        filesList = [k for k in filesList if 'M' in k]
+                        ##filesList = [k for k in filesList if 'M' in k]
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     filename = os.path.join( path, subfolder, file )
                     #Setting HDF5 File
                     fp = h5py.File(filename,'w')
                     #write metadata
                     self.writeMetadata(fp)
                     #Write data
                     grp = fp.create_group("Data")
                     ds = []
                     data = []
                     dsList = self.dsList
                     i = 0
                     while i < len(dsList):
                         dsInfo = dsList[i]
                         #One-dimension data
                         if dsInfo['mode'] == 0:
                             ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
                             ds.append(ds0)
                             data.append([])
                             i += 1
                             continue
                         elif dsInfo['mode'] == 2:
                             grp0 = grp.create_group(dsInfo['variable'])
                             ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
                             ds.append(ds0)
                             data.append([])
                             i += 1
                             continue
                         elif dsInfo['mode'] == 1:
                             grp0 = grp.create_group(dsInfo['variable'])
                             for j in range(dsInfo['dsNumber']):
                                 dsInfo = dsList[i]
                                 tableName = dsInfo['dsName']
                                 if dsInfo['nDim'] == 3:
                                     shape = dsInfo['shape'].astype(int)
                                     ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
                                 else:
                                     shape = int(dsInfo['shape'])
                                     ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
                                 ds.append(ds0)
                                 data.append([])
                                 i += 1
                     fp.flush()
                     fp.close()
                     log.log('creating file: {}'.format(filename), 'Writing')
                     self.filename = filename
                     self.ds = ds
                     self.data = data
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     if self.blockIndex == self.blocksPerFile or self.timeFlag():
                         self.setNextFile()
                     self.readBlock()
                     self.setBlock()     #Prepare data to be written
                     self.writeBlock()   #Write data
                     return
                 def readBlock(self):
                     '''
                     data Array configured
                     self.data
                     '''
                     dsList = self.dsList
                     ds = self.ds
                             #Setting HDF5 File
                     fp = h5py.File(self.filename,'r+')
                     grp = fp["Data"]
                     ind = 0
                     while ind < len(dsList):
                         dsInfo = dsList[ind]
                         if dsInfo['mode'] == 0:
                             ds0 = grp[dsInfo['variable']]
                             ds[ind] = ds0
                             ind += 1
                         else:
                             grp0 = grp[dsInfo['variable']]
                             for j in range(dsInfo['dsNumber']):
                                 dsInfo = dsList[ind]
                                 ds0 = grp0[dsInfo['dsName']]
                                 ds[ind] = ds0
                                 ind += 1
                     self.fp = fp
                     self.grp = grp
                     self.ds = ds
                     return
                 def setBlock(self):
                     '''
                     data Array configured
                     self.data
                     '''
                     #Creating Arrays
                     dsList = self.dsList
                     data = self.data
                     ind = 0
                     while ind < len(dsList):
                         dsInfo = dsList[ind]
                         dataAux = getattr(self.dataOut, dsInfo['variable'])
                         mode = dsInfo['mode']
                         nDim = dsInfo['nDim']
                         if mode == 0 or mode == 2 or nDim == 1:
                             data[ind] = dataAux
                             ind += 1
              #             elif nDim == 1:
              #                 data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
              #                 ind += 1
                         elif nDim == 2:
                             for j in range(dsInfo['dsNumber']):
                                 data[ind] = dataAux[j,:]
                                 ind += 1
                         elif nDim == 3:
                             for j in range(dsInfo['dsNumber']):
                                 data[ind] = dataAux[:,j,:]
                                 ind += 1
                     self.data = data
                     return
                 def writeBlock(self):
                     '''
                     Saves the block in the HDF5 file
                     '''
                     dsList = self.dsList
                     for i in range(len(self.ds)):
                         dsInfo = dsList[i]
                         nDim = dsInfo['nDim']
                         mode = dsInfo['mode']
                         #    First time
                         if self.firsttime:
                             if type(self.data[i]) == numpy.ndarray:
                                 if nDim == 3:
                                     self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
                                     self.ds[i].resize(self.data[i].shape)
                                 if mode == 2:
                                     self.ds[i].resize(self.data[i].shape)
                             self.ds[i][:] = self.data[i]
                         else:
                         #    From second time
                             #    Meteors!
                             if mode == 2:
                                 dataShape = self.data[i].shape
                                 dsShape = self.ds[i].shape
                                 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
                                 self.ds[i][dsShape[0]:,:] = self.data[i]
                             #    No dimension
                             elif mode == 0:
                                 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
                                 self.ds[i][0,-1] = self.data[i]
                             #    One dimension
                             elif nDim == 1:
                                 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
                                 self.ds[i][-1,:] = self.data[i]
                             #    Two dimension
                             elif nDim == 2:
                                 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
                                 self.ds[i][self.blockIndex,:] = self.data[i]
                             #    Three dimensions
                             elif nDim == 3:
                                 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
                                 self.ds[i][:,:,-1] = self.data[i]
                     self.firsttime = False
                     self.blockIndex += 1
                     #Close to save changes
                     self.fp.flush()
                     self.fp.close()
                     return
                 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
                     self.dataOut = dataOut
                     if not(self.isConfig):
                         self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList, mode=mode,
                                    setType=setType)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return
             @MPDecorator
             class ParameterReader(Reader, ProcessingUnit):
                 '''
                 Reads HDF5 format files
                 '''
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                     self.ext = ".hdf5"
                     self.optchar = "D"
                     self.timezone = "lt"
                     self.listMetaname = []
                     self.listMeta = []
                     self.listDataname = []
                     self.listData = []
                     self.listShapes = []
                     self.open_file = h5py.File
                     self.open_mode = 'r'
                     self.metadata = False
                     self.filefmt = "*%Y%j***"
                     self.folderfmt = "*%Y%j"
                 def setup(self, **kwargs):
                     self.set_kwargs(**kwargs)
                     if not self.ext.startswith('.'):
                         self.ext = '.{}'.format(self.ext)
                     if self.online:
                         log.log("Searching files in online mode...", self.name)
                         for nTries in range(self.nTries):
                             fullpath = self.searchFilesOnLine(self.path, self.startDate,
                                 self.endDate, self.expLabel, self.ext, self.walk,
                                 self.filefmt, self.folderfmt)
                             try:
                                 fullpath = next(fullpath)
                             except:
                                 fullpath = None
                             if fullpath:
                                 break
                             log.warning(
                                 'Waiting {} sec for a valid file in {}: try {} ...'.format(
                                     self.delay, self.path, nTries + 1),
                                 self.name)
                             time.sleep(self.delay)
                         if not(fullpath):
                             raise schainpy.admin.SchainError(
                                 'There isn\'t any valid file in {}'.format(self.path))
                         pathname, filename = os.path.split(fullpath)
                         self.year = int(filename[1:5])
                         self.doy = int(filename[5:8])
                         self.set = int(filename[8:11]) - 1
                     else:
                         log.log("Searching files in {}".format(self.path), self.name)
                         self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
                             self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
                     self.setNextFile()
                     return
                 def readFirstHeader(self):
                     '''Read metadata and data'''
                     self.__readMetadata()
                     self.__readData()
                     self.__setBlockList()
                     self.blockIndex = 0
                     return
                 def __setBlockList(self):
                     '''
                     Selects the data within the times defined
                     self.fp
                     self.startTime
                     self.endTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     startTime = self.startTime
                     endTime = self.endTime
                     index = self.listDataname.index('utctime')
                     thisUtcTime = self.listData[index]
                     self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     return
                 def __readMetadata(self):
                     '''
                     Reads Metadata
                     '''
                     listMetaname = []
                     listMetadata = []
                     if 'Metadata' in self.fp:
                         gp = self.fp['Metadata']
                         for item in list(gp.items()):
                             name = item[0]
                             if name=='variables':
                                 table = gp[name][:]
                                 listShapes = {}
                                 for shapes in table:
                                     listShapes[shapes[0].decode()] = numpy.array([shapes[1]])
                             else:
                                 data = gp[name].value
                                 listMetaname.append(name)
                                 listMetadata.append(data)
                     elif self.metadata:
                         metadata = json.loads(self.metadata)
                         listShapes = {}
                         for tup in metadata:
                             name, values, dim = tup
                             if dim == -1:
                                 listMetaname.append(name)
                                 listMetadata.append(self.fp[values].value)
                             else:
                                 listShapes[name] = numpy.array([dim])
                     else:
                         raise IOError('Missing Metadata group in file or metadata info')
                     self.listShapes = listShapes
                     self.listMetaname = listMetaname
                     self.listMeta = listMetadata
                     return
                 def __readData(self):
                     listdataname = []
                     listdata = []
                     if 'Data' in self.fp:
                         grp = self.fp['Data']
                         for item in list(grp.items()):
                             name = item[0]
                             listdataname.append(name)
                             dim = self.listShapes[name][0]
                             if dim == 0:
                                 array = grp[name].value
                             else:
                                 array = []
                                 for i in range(dim):
                                     array.append(grp[name]['table{:02d}'.format(i)].value)
                                 array = numpy.array(array)
                             listdata.append(array)
                     elif self.metadata:
                         metadata = json.loads(self.metadata)
                         for tup in metadata:
                             name, values, dim = tup
                             listdataname.append(name)
                             if dim == -1:
                                 continue
                             elif dim == 0:
                                 array = self.fp[values].value
                             else:
                                 array = []
                                 for var in values:
                                     array.append(self.fp[var].value)
                                 array = numpy.array(array)
                             listdata.append(array)
                     else:
                         raise IOError('Missing Data group in file or metadata info')
                     self.listDataname = listdataname
                     self.listData = listdata
                     return
                 def getData(self):
                     for i in range(len(self.listMeta)):
                         setattr(self.dataOut, self.listMetaname[i], self.listMeta[i])
                     for j in range(len(self.listData)):
                         dim = self.listShapes[self.listDataname[j]][0]
                         if dim == 0:
                             setattr(self.dataOut, self.listDataname[j], self.listData[j][self.blockIndex])
                         else:
                             setattr(self.dataOut, self.listDataname[j], self.listData[j][:,self.blockIndex])
                     self.dataOut.paramInterval = self.interval
                     self.dataOut.flagNoData = False
                     self.blockIndex += 1
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     if self.blockIndex == self.blocksPerFile:
                         self.setNextFile()
                     self.getData()
                     return
             @MPDecorator
             class ParameterWriter(Operation):
                 '''
                 HDF5 Writer, stores parameters data in HDF5 format files
                 path:             path where the files will be stored
                 blocksPerFile:    number of blocks that will be saved in per HDF5 format file
                 mode:             selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
                 metadataList:     list of attributes that will be stored as metadata
                 dataList:         list of attributes that will be stores as data
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 metaoptchar = "M"
                 metaFile = None
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 grp = None
                 ds = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 dsList = None   #List of dictionaries with dataset properties
                 tableDim = None
                 dtype = [('name', 'S20'),('nDim', 'i')]
                 currentDay = None
                 lastTime = None
                 def __init__(self):
                     Operation.__init__(self)
                     return
                 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataList = dataList
                     self.setType = setType
                     tableList = []
                     dsList = []
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         dataAux = getattr(self.dataOut, self.dataList[i])
                         dsDict['variable'] = self.dataList[i]
                         if dataAux is None:
                             continue
                         elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
                             dsDict['nDim'] = 0
                         else:
                             dsDict['nDim'] = len(dataAux.shape)
                             dsDict['shape'] = dataAux.shape
                             dsDict['dsNumber'] = dataAux.shape[0]
                         dsList.append(dsDict)
                         tableList.append((self.dataList[i], dsDict['nDim']))
                     self.dsList = dsList
                     self.tableDim = numpy.array(tableList, dtype=self.dtype)
                     self.currentDay = self.dataOut.datatime.date()
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     timeTuple = time.localtime(currentTime)
                     dataDay = timeTuple.tm_yday
                     if self.lastTime is None:
                         self.lastTime = currentTime
                         self.currentDay = dataDay
                         return False
                     timeDiff = currentTime - self.lastTime
                     #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                     if dataDay != self.currentDay:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > 3*60*60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, setType=None):
                     self.dataOut = dataOut
                     if not(self.isConfig):
                         self.setup(path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList,
                                    setType=setType)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join(path, subfolder)
                     if os.path.exists(fullpath):
                         filesList = os.listdir(fullpath)
                         filesList = [k for k in filesList if k.startswith(self.optchar)]
                         if len( filesList ) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     self.filename = os.path.join( path, subfolder, file )
                     #Setting HDF5 File
                     self.fp = h5py.File(self.filename, 'w')
                     #write metadata
                     self.writeMetadata(self.fp)
                     #Write data
                     self.writeData(self.fp)
                 def writeMetadata(self, fp):
                     grp = fp.create_group("Metadata")
                     grp.create_dataset('variables', data=self.tableDim, dtype=self.dtype)
                     for i in range(len(self.metadataList)):
                         if not hasattr(self.dataOut, self.metadataList[i]):
                             log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
                             continue
                         value = getattr(self.dataOut, self.metadataList[i])
                         grp.create_dataset(self.metadataList[i], data=value)
                     return
                 def writeData(self, fp):
                     grp = fp.create_group("Data")
                     dtsets = []
                     data = []
                     for dsInfo in self.dsList:
                         if dsInfo['nDim'] == 0:
                             ds = grp.create_dataset(
                                 dsInfo['variable'],
                                 (self.blocksPerFile, ),
                                 chunks=True,
                                 dtype=numpy.float64)
                             dtsets.append(ds)
                             data.append((dsInfo['variable'], -1))
                         else:
                             sgrp = grp.create_group(dsInfo['variable'])
                             for i in range(dsInfo['dsNumber']):
                                 ds = sgrp.create_dataset(
                                     'table{:02d}'.format(i),
                                     (self.blocksPerFile, ) + dsInfo['shape'][1:],
                                     chunks=True)
                                 dtsets.append(ds)
                                 data.append((dsInfo['variable'], i))
                     fp.flush()
                     log.log('Creating file: {}'.format(fp.filename), self.name)
                     self.ds = dtsets
                     self.data = data
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
                         self.closeFile()
                         self.setNextFile()
                     for i, ds in enumerate(self.ds):
                         attr, ch = self.data[i]
                         if ch == -1:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)
                         else:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
                     self.fp.flush()
                     self.blockIndex += 1
                     log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
                     return
                 def closeFile(self):
                     if self.blockIndex != self.blocksPerFile:
                         for ds in self.ds:
                             ds.resize(self.blockIndex, axis=0)
                     self.fp.flush()
                     self.fp.close()
                 def close(self):
                     self.closeFile()

schainpy/model/proc/jroproc_base.py +37 -40

             '''
             Updated for multiprocessing
             Author : Sergio Cortez
             Jan 2018
             Abstract:
                 Base class for processing units and operations. A decorator provides multiprocessing features and interconnect the processes created.
                 The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
                 The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
             Based on:
                 $Author: murco $
                 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
             '''
             import os
             import sys
             import inspect
             import zmq
             import time
             import pickle
             import traceback
             try:
                 from queue import Queue
             except:
                 from Queue import Queue
             from threading import Thread
             from multiprocessing import Process
             from schainpy.utils import log
             class ProcessingUnit(object):
                 """
                 Update - Jan 2018 - MULTIPROCESSING
                 All the "call" methods present in the previous base were removed.
                 The majority of operations are independant processes, thus
                 the decorator is in charge of communicate the operation processes
                 with the proccessing unit via IPC.
                 The constructor does not receive any argument. The remaining methods
                 are related with the operations to execute.
                 """
                 proc_type = 'processing'
                 __attrs__ = []
                 def __init__(self):
                     self.dataIn = None
                     self.dataOut = None
                     self.isConfig = False
                     self.operations = []
                     self.plots = []
                 def getAllowedArgs(self):
                     if hasattr(self, '__attrs__'):
                         return self.__attrs__
                     else:
                         return inspect.getargspec(self.run).args
                 def addOperation(self, conf, operation):
                     """
                     This method is used in the controller, and update the dictionary containing the operations to execute. The dict
                     posses the id of the operation process (IPC purposes)
                         Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
                         identificador asociado a este objeto.
                         Input:
                             object    :    objeto de la clase "Operation"
                         Return:
                             objId    :    identificador del objeto, necesario para comunicar con master(procUnit)
                     """
                     self.operations.append(
                         (operation, conf.type, conf.id, conf.getKwargs()))
                     if 'plot' in self.name.lower():
                         self.plots.append(operation.CODE)
                 def getOperationObj(self, objId):
                     if objId not in list(self.operations.keys()):
                         return None
                     return self.operations[objId]
                 def operation(self, **kwargs):
                     """
                     Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
                     atributos del objeto dataOut
                     Input:
                         **kwargs    :    Diccionario de argumentos de la funcion a ejecutar
                     """
                     raise NotImplementedError
                 def setup(self):
                     raise NotImplementedError
                 def run(self):
                     raise NotImplementedError
                 def close(self):
                     return
             class Operation(object):
                 """
                 Update - Jan 2018 - MULTIPROCESSING
                 Most of the methods remained the same. The decorator parse the arguments and executed the run() method for each process.
                 The constructor doe snot receive any argument, neither the baseclass.
                     Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
                     y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
                     acumulacion dentro de esta clase
                     Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
                 """
                 proc_type = 'operation'
                 __attrs__ = []
                 def __init__(self):
                     self.id = None
                     self.isConfig = False
                     if not hasattr(self, 'name'):
                         self.name = self.__class__.__name__
                 def getAllowedArgs(self):
                     if hasattr(self, '__attrs__'):
                         return self.__attrs__
                     else:
                         return inspect.getargspec(self.run).args
                 def setup(self):
                     self.isConfig = True
                     raise NotImplementedError
                 def run(self, dataIn, **kwargs):
                     """
                     Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
                     atributos del objeto dataIn.
                     Input:
                         dataIn    :    objeto del tipo JROData
                     Return:
                         None
                     Affected:
                         __buffer    :    buffer de recepcion de datos.
                     """
                     if not self.isConfig:
                         self.setup(**kwargs)
                     raise NotImplementedError
                 def close(self):
                     return
             class InputQueue(Thread):
                 '''
                 Class to hold input data for Proccessing Units and external Operations,
                 '''
                 def __init__(self, project_id, inputId, lock=None):
                     Thread.__init__(self)
                     self.queue = Queue()
                     self.project_id = project_id
                     self.inputId = inputId
                     self.lock = lock
                     self.islocked = False
                     self.size = 0
                 def run(self):
                     c = zmq.Context()
                     self.receiver = c.socket(zmq.SUB)
                     self.receiver.connect(
                         'ipc:///tmp/schain/{}_pub'.format(self.project_id))
                     self.receiver.setsockopt(zmq.SUBSCRIBE, self.inputId.encode())
                     while True:
                         obj = self.receiver.recv_multipart()[1]
                         self.size += sys.getsizeof(obj)
                         self.queue.put(obj)
                 def get(self):
                     if not self.islocked and self.size/1000000 > 512:
                         self.lock.n.value += 1
                         self.islocked = True
                         self.lock.clear()
                     elif self.islocked and self.size/1000000 <= 512:
                         self.islocked = False
                         self.lock.n.value -= 1
                         if self.lock.n.value == 0:
                             self.lock.set()
                     obj = self.queue.get()
                     self.size -= sys.getsizeof(obj)
                     return pickle.loads(obj)
             def MPDecorator(BaseClass):
                 """
                 Multiprocessing class decorator
                 This function add multiprocessing features to a BaseClass. Also, it handle
                 the communication beetween processes (readers, procUnits and operations).
                 """
                 class MPClass(BaseClass, Process):
                     def __init__(self, *args, **kwargs):
                         super(MPClass, self).__init__()
                         Process.__init__(self)
                         self.operationKwargs = {}
                         self.args = args
                         self.kwargs = kwargs
                         self.sender = None
                         self.receiver = None
                         self.i = 0
                         self.t = time.time()
                         self.name = BaseClass.__name__
                         self.__doc__ = BaseClass.__doc__
                         if 'plot' in self.name.lower() and not self.name.endswith('_'):
                             self.name = '{}{}'.format(self.CODE.upper(), 'Plot')
                         self.start_time = time.time()
                         self.id = args[0]
                         self.inputId = args[1]
                         self.project_id = args[2]
                         self.err_queue = args[3]
                         self.lock = args[4]
                         self.typeProc = args[5]
                         self.err_queue.put('#_start_#')
                         if self.inputId is not None:
                             self.queue = InputQueue(self.project_id, self.inputId, self.lock)
                     def subscribe(self):
                         '''
                         Start the zmq socket receiver and subcribe to input ID.
                         '''
                         self.queue.start()
                     def listen(self):
                         '''
                         This function waits for objects
                         '''
                         return self.queue.get()
                     def set_publisher(self):
                         '''
                         This function create a zmq socket for publishing objects.
                         '''
                         time.sleep(0.5)
                         c = zmq.Context()
                         self.sender = c.socket(zmq.PUB)
                         self.sender.connect(
                             'ipc:///tmp/schain/{}_sub'.format(self.project_id))
                     def publish(self, data, id):
                         '''
                         This function publish an object, to an specific topic.
                         It blocks publishing when receiver queue is full to avoid data loss
                         '''
                         if self.inputId is None:
                             self.lock.wait()
                         self.sender.send_multipart([str(id).encode(), pickle.dumps(data)])
                     def runReader(self):
                         '''
                         Run fuction for read units
                         '''
                         while True:
                             try:
                                 BaseClass.run(self, **self.kwargs)
                             except:
                                 err = traceback.format_exc()
                                 if 'No more files' in err:
                                     log.warning('No more files to read', self.name)
                                 else:
                                     self.err_queue.put('{}|{}'.format(self.name, err))
                                 self.dataOut.error = True
                             for op, optype, opId, kwargs in self.operations:
                                 if optype == 'self' and not self.dataOut.flagNoData:
                                     op(**kwargs)
                                 elif optype == 'other' and not self.dataOut.flagNoData:
                                     self.dataOut = op.run(self.dataOut, **self.kwargs)
                                 elif optype == 'external':
                                     self.publish(self.dataOut, opId)
                             if self.dataOut.flagNoData and not self.dataOut.error:
                                 continue
                             self.publish(self.dataOut, self.id)
+                            if self.dataOut.error:
-                            if self.dataOut.error:
                                 break
                         time.sleep(0.5)
                     def runProc(self):
                         '''
                         Run function for proccessing units
                         '''
                         while True:
                             self.dataIn = self.listen()
                             if self.dataIn.flagNoData and self.dataIn.error is None:
                                 continue
                             elif not self.dataIn.error:
                                 try:
                                     BaseClass.run(self, **self.kwargs)
                                 except:
                                     self.err_queue.put('{}|{}'.format(self.name, traceback.format_exc()))
                                     self.dataOut.error = True
                             elif self.dataIn.error:
                                 self.dataOut.error = self.dataIn.error
                                 self.dataOut.flagNoData = True
                             for op, optype, opId, kwargs in self.operations:
                                 if optype == 'self' and not self.dataOut.flagNoData:
                                     op(**kwargs)
                                 elif optype == 'other' and not self.dataOut.flagNoData:
                                     self.dataOut = op.run(self.dataOut, **kwargs)
                                 elif optype == 'external' and not self.dataOut.flagNoData:
                                     self.publish(self.dataOut, opId)
                             self.publish(self.dataOut, self.id)
                             for op, optype, opId, kwargs in self.operations:
                                 if optype == 'external' and self.dataOut.error:
                                     self.publish(self.dataOut, opId)
                             if self.dataOut.error:
                                 break
                         time.sleep(0.5)
                     def runOp(self):
                         '''
                         Run function for external operations (this operations just receive data
                         ex: plots, writers, publishers)
                         '''
                         while True:
                             dataOut = self.listen()
                             if not dataOut.error:
                                 try:
                                     BaseClass.run(self, dataOut, **self.kwargs)
                                 except:
                                     self.err_queue.put('{}|{}'.format(self.name, traceback.format_exc()))
                                     dataOut.error = True
                             else:
                                 break
                     def run(self):
                         if self.typeProc is "ProcUnit":
                             if self.inputId is not None:
                                 self.subscribe()
                             self.set_publisher()
                             if 'Reader' not in BaseClass.__name__:
                                 self.runProc()
                             else:
                                 self.runReader()
                         elif self.typeProc is "Operation":
                             self.subscribe()
                             self.runOp()
                         else:
                             raise ValueError("Unknown type")
                         self.close()
                     def close(self):
                         BaseClass.close(self)
                         self.err_queue.put('#_end_#')
                         if self.sender:
                             self.sender.close()
                         if self.receiver:
                             self.receiver.close()
                         log.success('Done...(Time:{:4.2f} secs)'.format(time.time()-self.start_time), self.name)
                 return MPClass

schainpy/model/proc/jroproc_parameters.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/jroproc_spectra.py +31 -31

             import itertools
             import numpy
             from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
             from schainpy.model.data.jrodata import Spectra
             from schainpy.model.data.jrodata import hildebrand_sekhon
             from schainpy.utils import log
             @MPDecorator
             class SpectraProc(ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.buffer = None
                     self.firstdatatime = None
                     self.profIndex = 0
                     self.dataOut = Spectra()
                     self.id_min = None
                     self.id_max = None
                     self.setupReq = False #Agregar a todas las unidades de proc
                 def __updateSpecFromVoltage(self):
                     self.dataOut.timeZone = self.dataIn.timeZone
                     self.dataOut.dstFlag = self.dataIn.dstFlag
                     self.dataOut.errorCount = self.dataIn.errorCount
                     self.dataOut.useLocalTime = self.dataIn.useLocalTime
                     try:
                         self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
                     except:
                         pass
                     self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
                     self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
                     self.dataOut.channelList = self.dataIn.channelList
                     self.dataOut.heightList = self.dataIn.heightList
                     self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                     self.dataOut.nBaud = self.dataIn.nBaud
                     self.dataOut.nCode = self.dataIn.nCode
                     self.dataOut.code = self.dataIn.code
                     self.dataOut.nProfiles = self.dataOut.nFFTPoints
                     self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
                     self.dataOut.utctime = self.firstdatatime
                     # asumo q la data esta decodificada
                     self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
                     # asumo q la data esta sin flip
                     self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.nCohInt = self.dataIn.nCohInt
                     self.dataOut.nIncohInt = 1
                     self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
                     self.dataOut.frequency = self.dataIn.frequency
                     self.dataOut.realtime = self.dataIn.realtime
                     self.dataOut.azimuth = self.dataIn.azimuth
                     self.dataOut.zenith = self.dataIn.zenith
                     self.dataOut.beam.codeList = self.dataIn.beam.codeList
                     self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
                     self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
                 def __getFft(self):
                     """
                     Convierte valores de Voltaje a Spectra
                     Affected:
                         self.dataOut.data_spc
                         self.dataOut.data_cspc
                         self.dataOut.data_dc
                         self.dataOut.heightList
                         self.profIndex
                         self.buffer
                         self.dataOut.flagNoData
                     """
                     fft_volt = numpy.fft.fft(
                         self.buffer, n=self.dataOut.nFFTPoints, axis=1)
                     fft_volt = fft_volt.astype(numpy.dtype('complex'))
                     dc = fft_volt[:, 0, :]
                     # calculo de self-spectra
                     fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
                     spc = fft_volt * numpy.conjugate(fft_volt)
                     spc = spc.real
                     blocksize = 0
                     blocksize += dc.size
                     blocksize += spc.size
                     cspc = None
                     pairIndex = 0
                     if self.dataOut.pairsList != None:
                         # calculo de cross-spectra
                         cspc = numpy.zeros(
                             (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
                         for pair in self.dataOut.pairsList:
                             if pair[0] not in self.dataOut.channelList:
                                 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
                                     str(pair), str(self.dataOut.channelList)))
                             if pair[1] not in self.dataOut.channelList:
                                 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
                                     str(pair), str(self.dataOut.channelList)))
                             cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
                                 numpy.conjugate(fft_volt[pair[1], :, :])
                             pairIndex += 1
                         blocksize += cspc.size
                     self.dataOut.data_spc = spc
                     self.dataOut.data_cspc = cspc
                     self.dataOut.data_dc = dc
                     self.dataOut.blockSize = blocksize
                     self.dataOut.flagShiftFFT = True
                 def run(self, nProfiles=None, nFFTPoints=None, pairsList=[], ippFactor=None, shift_fft=False):
                     if self.dataIn.type == "Spectra":
                         self.dataOut.copy(self.dataIn)
                         if shift_fft:
                             #desplaza a la derecha en el eje 2 determinadas posiciones
                             shift = int(self.dataOut.nFFTPoints/2)
                             self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
                             if self.dataOut.data_cspc is not None:
                                 #desplaza a la derecha en el eje 2 determinadas posiciones
                                 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
                         return True
                     if self.dataIn.type == "Voltage":
                         self.dataOut.flagNoData = True
                         if nFFTPoints == None:
                             raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
                         if nProfiles == None:
                             nProfiles = nFFTPoints
                         if ippFactor == None:
                             ippFactor = 1
                         self.dataOut.ippFactor = ippFactor
                         self.dataOut.nFFTPoints = nFFTPoints
                         self.dataOut.pairsList = pairsList
                         if self.buffer is None:
                             self.buffer = numpy.zeros((self.dataIn.nChannels,
                                                        nProfiles,
                                                        self.dataIn.nHeights),
                                                       dtype='complex')
                         if self.dataIn.flagDataAsBlock:
                             nVoltProfiles = self.dataIn.data.shape[1]
                             if nVoltProfiles == nProfiles:
                                 self.buffer = self.dataIn.data.copy()
                                 self.profIndex = nVoltProfiles
                             elif nVoltProfiles < nProfiles:
                                 if self.profIndex == 0:
                                     self.id_min = 0
                                     self.id_max = nVoltProfiles
                                 self.buffer[:, self.id_min:self.id_max,
                                             :] = self.dataIn.data
                                 self.profIndex += nVoltProfiles
                                 self.id_min += nVoltProfiles
                                 self.id_max += nVoltProfiles
                             else:
                                 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
                                     self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
                                 self.dataOut.flagNoData = True
                                 return 0
                         else:
                             self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
                             self.profIndex += 1
                         if self.firstdatatime == None:
                             self.firstdatatime = self.dataIn.utctime
                         if self.profIndex == nProfiles:
                             self.__updateSpecFromVoltage()
                             self.__getFft()
                             self.dataOut.flagNoData = False
                             self.firstdatatime = None
                             self.profIndex = 0
                         return True
                     raise ValueError("The type of input object '%s' is not valid" % (
                         self.dataIn.type))
                 def __selectPairs(self, pairsList):
                     if not pairsList:
                         return
                     pairs = []
                     pairsIndex = []
                     for pair in pairsList:
                         if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
                             continue
                         pairs.append(pair)
                         pairsIndex.append(pairs.index(pair))
                     self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
                     self.dataOut.pairsList = pairs
                     return
                 def __selectPairsByChannel(self, channelList=None):
                     if channelList == None:
                         return
                     pairsIndexListSelected = []
                     for pairIndex in self.dataOut.pairsIndexList:
                         # First pair
                         if self.dataOut.pairsList[pairIndex][0] not in channelList:
                             continue
                         # Second pair
                         if self.dataOut.pairsList[pairIndex][1] not in channelList:
                             continue
                         pairsIndexListSelected.append(pairIndex)
                     if not pairsIndexListSelected:
                         self.dataOut.data_cspc = None
                         self.dataOut.pairsList = []
                         return
                     self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndexListSelected]
                     self.dataOut.pairsList = [self.dataOut.pairsList[i]
                                               for i in pairsIndexListSelected]
                     return
                 def selectChannels(self, channelList):
                     channelIndexList = []
                     for channel in channelList:
                         if channel not in self.dataOut.channelList:
                             raise ValueError("Error selecting channels, Channel %d is not valid.\nAvailable channels = %s" % (
                                 channel, str(self.dataOut.channelList)))
                         index = self.dataOut.channelList.index(channel)
                         channelIndexList.append(index)
                     self.selectChannelsByIndex(channelIndexList)
                 def selectChannelsByIndex(self, channelIndexList):
                     """
                     Selecciona un bloque de datos en base a canales segun el channelIndexList
                     Input:
                         channelIndexList    :    lista sencilla de canales a seleccionar por ej. [2,3,7]
                     Affected:
                         self.dataOut.data_spc
                         self.dataOut.channelIndexList
                         self.dataOut.nChannels
                     Return:
                         None
                     """
                     for channelIndex in channelIndexList:
                         if channelIndex not in self.dataOut.channelIndexList:
                             raise ValueError("Error selecting channels: The value %d in channelIndexList is not valid.\nAvailable channel indexes = " % (
                                 channelIndex, self.dataOut.channelIndexList))
                     data_spc = self.dataOut.data_spc[channelIndexList, :]
                     data_dc = self.dataOut.data_dc[channelIndexList, :]
                     self.dataOut.data_spc = data_spc
                     self.dataOut.data_dc = data_dc
                     # self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
                     self.dataOut.channelList = range(len(channelIndexList))
                     self.__selectPairsByChannel(channelIndexList)
                     return 1
                 def selectFFTs(self, minFFT, maxFFT ):
                     """
                     Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
                     minFFT<= FFT <= maxFFT
                     """
                     if (minFFT > maxFFT):
                         raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
                     if (minFFT < self.dataOut.getFreqRange()[0]):
                         minFFT = self.dataOut.getFreqRange()[0]
                     if (maxFFT > self.dataOut.getFreqRange()[-1]):
                         maxFFT = self.dataOut.getFreqRange()[-1]
                     minIndex = 0
                     maxIndex = 0
                     FFTs = self.dataOut.getFreqRange()
                     inda = numpy.where(FFTs >= minFFT)
                     indb = numpy.where(FFTs <= maxFFT)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(FFTs)
                     self.selectFFTsByIndex(minIndex, maxIndex)
                     return 1
                 def setH0(self, h0, deltaHeight = None):
                     if not deltaHeight:
                         deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     nHeights = self.dataOut.nHeights
                     newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
                     self.dataOut.heightList = newHeiRange
                 def selectHeights(self, minHei, maxHei):
                     """
                     Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
                     minHei <= height <= maxHei
                     Input:
                         minHei    :    valor minimo de altura a considerar
                         maxHei    :    valor maximo de altura a considerar
                     Affected:
                         Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if (minHei > maxHei):
                         raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minHei, maxHei))
                     if (minHei < self.dataOut.heightList[0]):
                         minHei = self.dataOut.heightList[0]
                     if (maxHei > self.dataOut.heightList[-1]):
                         maxHei = self.dataOut.heightList[-1]
                     minIndex = 0
                     maxIndex = 0
                     heights = self.dataOut.heightList
                     inda = numpy.where(heights >= minHei)
                     indb = numpy.where(heights <= maxHei)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(heights)
                     self.selectHeightsByIndex(minIndex, maxIndex)
                     return 1
                 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
                     newheis = numpy.where(
                         self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
                     if hei_ref != None:
                         newheis = numpy.where(self.dataOut.heightList > hei_ref)
                     minIndex = min(newheis[0])
                     maxIndex = max(newheis[0])
                     data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
                     heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
                     # determina indices
                     nheis = int(self.dataOut.radarControllerHeaderObj.txB /
                                 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
                     avg_dB = 10 * \
                         numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
                     beacon_dB = numpy.sort(avg_dB)[-nheis:]
                     beacon_heiIndexList = []
                     for val in avg_dB.tolist():
                         if val >= beacon_dB[0]:
                             beacon_heiIndexList.append(avg_dB.tolist().index(val))
                     #data_spc = data_spc[:,:,beacon_heiIndexList]
                     data_cspc = None
                     if self.dataOut.data_cspc is not None:
                         data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
                         #data_cspc = data_cspc[:,:,beacon_heiIndexList]
                     data_dc = None
                     if self.dataOut.data_dc is not None:
                         data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
                         #data_dc = data_dc[:,beacon_heiIndexList]
                     self.dataOut.data_spc = data_spc
                     self.dataOut.data_cspc = data_cspc
                     self.dataOut.data_dc = data_dc
                     self.dataOut.heightList = heightList
                     self.dataOut.beacon_heiIndexList = beacon_heiIndexList
                     return 1
                 def selectFFTsByIndex(self, minIndex, maxIndex):
                     """
                     """
                     if (minIndex < 0) or (minIndex > maxIndex):
                         raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex))
                     if (maxIndex >= self.dataOut.nProfiles):
                         maxIndex = self.dataOut.nProfiles-1
                     #Spectra
                     data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:]
                     data_cspc = None
                     if self.dataOut.data_cspc is not None:
                         data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:]
                     data_dc = None
                     if self.dataOut.data_dc is not None:
                         data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:]
                     self.dataOut.data_spc = data_spc
                     self.dataOut.data_cspc = data_cspc
                     self.dataOut.data_dc = data_dc
                     self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
                     self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
                     self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
                     return 1
                 def selectHeightsByIndex(self, minIndex, maxIndex):
                     """
                     Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
                     minIndex <= index <= maxIndex
                     Input:
                         minIndex    :    valor de indice minimo de altura a considerar
                         maxIndex    :    valor de indice maximo de altura a considerar
                     Affected:
                         self.dataOut.data_spc
                         self.dataOut.data_cspc
                         self.dataOut.data_dc
                         self.dataOut.heightList
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if (minIndex < 0) or (minIndex > maxIndex):
                         raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (
                             minIndex, maxIndex))
                     if (maxIndex >= self.dataOut.nHeights):
                         maxIndex = self.dataOut.nHeights - 1
                     # Spectra
                     data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
                     data_cspc = None
                     if self.dataOut.data_cspc is not None:
                         data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
                     data_dc = None
                     if self.dataOut.data_dc is not None:
                         data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
                     self.dataOut.data_spc = data_spc
                     self.dataOut.data_cspc = data_cspc
                     self.dataOut.data_dc = data_dc
                     self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
                     return 1
                 def removeDC(self, mode=2):
                     jspectra = self.dataOut.data_spc
                     jcspectra = self.dataOut.data_cspc
                     num_chan = jspectra.shape[0]
                     num_hei = jspectra.shape[2]
                     if jcspectra is not None:
                         jcspectraExist = True
                         num_pairs = jcspectra.shape[0]
                     else:
                         jcspectraExist = False
                     freq_dc = int(jspectra.shape[1] / 2)
                     ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
                     ind_vel = ind_vel.astype(int)
                     if ind_vel[0] < 0:
                         ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
                     if mode == 1:
                         jspectra[:, freq_dc, :] = (
                             jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2  # CORRECCION
                         if jcspectraExist:
                             jcspectra[:, freq_dc, :] = (
                                 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
                     if mode == 2:
                         vel = numpy.array([-2, -1, 1, 2])
                         xx = numpy.zeros([4, 4])
                         for fil in range(4):
                             xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
                         xx_inv = numpy.linalg.inv(xx)
                         xx_aux = xx_inv[0, :]
                         for ich in range(num_chan):
                             yy = jspectra[ich, ind_vel, :]
                             jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
                             junkid = jspectra[ich, freq_dc, :] <= 0
                             cjunkid = sum(junkid)
                             if cjunkid.any():
                                 jspectra[ich, freq_dc, junkid.nonzero()] = (
                                     jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
                         if jcspectraExist:
                             for ip in range(num_pairs):
                                 yy = jcspectra[ip, ind_vel, :]
                                 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
                     self.dataOut.data_spc = jspectra
                     self.dataOut.data_cspc = jcspectra
                     return 1
                 def removeInterference2(self):
                     cspc = self.dataOut.data_cspc
                     spc = self.dataOut.data_spc
                     Heights = numpy.arange(cspc.shape[2])
                     realCspc = numpy.abs(cspc)
                     for i in range(cspc.shape[0]):
                         LinePower= numpy.sum(realCspc[i], axis=0)
                         Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
                         SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
                         InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
                         InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
                         InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
                         InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
                         #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
                         if len(InterferenceRange)<int(cspc.shape[1]*0.3):
                             cspc[i,InterferenceRange,:] = numpy.NaN
                     self.dataOut.data_cspc = cspc
                 def removeInterference(self,  interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None):
                     jspectra = self.dataOut.data_spc
                     jcspectra = self.dataOut.data_cspc
                     jnoise = self.dataOut.getNoise()
                     num_incoh = self.dataOut.nIncohInt
                     num_channel = jspectra.shape[0]
                     num_prof = jspectra.shape[1]
                     num_hei = jspectra.shape[2]
                     # hei_interf
                     if hei_interf is None:
                         count_hei = int(num_hei / 2)
                         hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
                         hei_interf = numpy.asarray(hei_interf)[0]
                     # nhei_interf
                     if (nhei_interf == None):
                         nhei_interf = 5
                     if (nhei_interf < 1):
                         nhei_interf = 1
                     if (nhei_interf > count_hei):
                         nhei_interf = count_hei
                     if (offhei_interf == None):
                         offhei_interf = 0
                     ind_hei = list(range(num_hei))
             #         mask_prof = numpy.asarray(range(num_prof - 2)) + 1
             #         mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
                     mask_prof = numpy.asarray(list(range(num_prof)))
                     num_mask_prof = mask_prof.size
                     comp_mask_prof = [0, num_prof / 2]
                     # noise_exist:    Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
                     if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
                         jnoise = numpy.nan
                     noise_exist = jnoise[0] < numpy.Inf
                     # Subrutina de Remocion de la Interferencia
                     for ich in range(num_channel):
                         # Se ordena los espectros segun su potencia (menor a mayor)
                         power = jspectra[ich, mask_prof, :]
                         power = power[:, hei_interf]
                         power = power.sum(axis=0)
                         psort = power.ravel().argsort()
                         # Se estima la interferencia promedio en los Espectros de Potencia empleando
                         junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
                             offhei_interf, nhei_interf + offhei_interf))]]]
                         if noise_exist:
                             #    tmp_noise = jnoise[ich] / num_prof
                             tmp_noise = jnoise[ich]
                         junkspc_interf = junkspc_interf - tmp_noise
                         #junkspc_interf[:,comp_mask_prof] = 0
                         jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
                         jspc_interf = jspc_interf.transpose()
                         # Calculando el espectro de interferencia promedio
                         noiseid = numpy.where(
                             jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
                         noiseid = noiseid[0]
                         cnoiseid = noiseid.size
                         interfid = numpy.where(
                             jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
                         interfid = interfid[0]
                         cinterfid = interfid.size
                         if (cnoiseid > 0):
                             jspc_interf[noiseid] = 0
                         # Expandiendo los perfiles a limpiar
                         if (cinterfid > 0):
                             new_interfid = (
                                 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
                             new_interfid = numpy.asarray(new_interfid)
                             new_interfid = {x for x in new_interfid}
                             new_interfid = numpy.array(list(new_interfid))
                             new_cinterfid = new_interfid.size
                         else:
                             new_cinterfid = 0
                         for ip in range(new_cinterfid):
                             ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
                             jspc_interf[new_interfid[ip]
                                         ] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]]
                         jspectra[ich, :, ind_hei] = jspectra[ich, :,
                                                              ind_hei] - jspc_interf  # Corregir indices
                         # Removiendo la interferencia del punto de mayor interferencia
                         ListAux = jspc_interf[mask_prof].tolist()
                         maxid = ListAux.index(max(ListAux))
                         if cinterfid > 0:
                             for ip in range(cinterfid * (interf == 2) - 1):
                                 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
                                        (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
                                 cind = len(ind)
                                 if (cind > 0):
                                     jspectra[ich, interfid[ip], ind] = tmp_noise * \
                                         (1 + (numpy.random.uniform(cind) - 0.5) /
                                          numpy.sqrt(num_incoh))
                             ind = numpy.array([-2, -1, 1, 2])
                             xx = numpy.zeros([4, 4])
                             for id1 in range(4):
                                 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
                             xx_inv = numpy.linalg.inv(xx)
                             xx = xx_inv[:, 0]
                             ind = (ind + maxid + num_mask_prof) % num_mask_prof
                             yy = jspectra[ich, mask_prof[ind], :]
                             jspectra[ich, mask_prof[maxid], :] = numpy.dot(
                                 yy.transpose(), xx)
                         indAux = (jspectra[ich, :, :] < tmp_noise *
                                   (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
                         jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
                             (1 - 1 / numpy.sqrt(num_incoh))
                     # Remocion de Interferencia en el Cross Spectra
                     if jcspectra is None:
                         return jspectra, jcspectra
                     num_pairs = int(jcspectra.size / (num_prof * num_hei))
                     jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
                     for ip in range(num_pairs):
                         #-------------------------------------------
                         cspower = numpy.abs(jcspectra[ip, mask_prof, :])
                         cspower = cspower[:, hei_interf]
                         cspower = cspower.sum(axis=0)
                         cspsort = cspower.ravel().argsort()
                         junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
                             offhei_interf, nhei_interf + offhei_interf))]]]
                         junkcspc_interf = junkcspc_interf.transpose()
                         jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
                         ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
                         median_real = int(numpy.median(numpy.real(
                             junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
                         median_imag = int(numpy.median(numpy.imag(
                             junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
                         comp_mask_prof = [int(e) for e in comp_mask_prof]
                         junkcspc_interf[comp_mask_prof, :] = numpy.complex(
                             median_real, median_imag)
                         for iprof in range(num_prof):
                             ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
                             jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]]
                         # Removiendo la Interferencia
                         jcspectra[ip, :, ind_hei] = jcspectra[ip,
                                                               :, ind_hei] - jcspc_interf
                         ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
                         maxid = ListAux.index(max(ListAux))
                         ind = numpy.array([-2, -1, 1, 2])
                         xx = numpy.zeros([4, 4])
                         for id1 in range(4):
                             xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
                         xx_inv = numpy.linalg.inv(xx)
                         xx = xx_inv[:, 0]
                         ind = (ind + maxid + num_mask_prof) % num_mask_prof
                         yy = jcspectra[ip, mask_prof[ind], :]
                         jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
                     # Guardar Resultados
                     self.dataOut.data_spc = jspectra
                     self.dataOut.data_cspc = jcspectra
                     return 1
                 def setRadarFrequency(self, frequency=None):
                     if frequency != None:
                         self.dataOut.frequency = frequency
                     return 1
                 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
                     # validacion de rango
                     if minHei == None:
                         minHei = self.dataOut.heightList[0]
                     if maxHei == None:
                         maxHei = self.dataOut.heightList[-1]
                     if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
                         print('minHei: %.2f is out of the heights range' % (minHei))
                         print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
                         minHei = self.dataOut.heightList[0]
                     if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
                         print('maxHei: %.2f is out of the heights range' % (maxHei))
                         print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
                         maxHei = self.dataOut.heightList[-1]
                     # validacion de velocidades
                     velrange = self.dataOut.getVelRange(1)
                     if minVel == None:
                         minVel = velrange[0]
                     if maxVel == None:
                         maxVel = velrange[-1]
                     if (minVel < velrange[0]) or (minVel > maxVel):
                         print('minVel: %.2f is out of the velocity range' % (minVel))
                         print('minVel is setting to %.2f' % (velrange[0]))
                         minVel = velrange[0]
                     if (maxVel > velrange[-1]) or (maxVel < minVel):
                         print('maxVel: %.2f is out of the velocity range' % (maxVel))
                         print('maxVel is setting to %.2f' % (velrange[-1]))
                         maxVel = velrange[-1]
                     # seleccion de indices para rango
                     minIndex = 0
                     maxIndex = 0
                     heights = self.dataOut.heightList
                     inda = numpy.where(heights >= minHei)
                     indb = numpy.where(heights <= maxHei)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(heights)
                     if (minIndex < 0) or (minIndex > maxIndex):
                         raise ValueError("some value in (%d,%d) is not valid" % (
                             minIndex, maxIndex))
                     if (maxIndex >= self.dataOut.nHeights):
                         maxIndex = self.dataOut.nHeights - 1
                     # seleccion de indices para velocidades
                     indminvel = numpy.where(velrange >= minVel)
                     indmaxvel = numpy.where(velrange <= maxVel)
                     try:
                         minIndexVel = indminvel[0][0]
                     except:
                         minIndexVel = 0
                     try:
                         maxIndexVel = indmaxvel[0][-1]
                     except:
                         maxIndexVel = len(velrange)
                     # seleccion del espectro
                     data_spc = self.dataOut.data_spc[:,
                                                      minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
                     # estimacion de ruido
                     noise = numpy.zeros(self.dataOut.nChannels)
                     for channel in range(self.dataOut.nChannels):
                         daux = data_spc[channel, :, :]
                         noise[channel] = hildebrand_sekhon(daux, self.dataOut.nIncohInt)
                     self.dataOut.noise_estimation = noise.copy()
                     return 1
             class IncohInt(Operation):
                 __profIndex = 0
                 __withOverapping = False
                 __byTime = False
                 __initime = None
                 __lastdatatime = None
                 __integrationtime = None
                 __buffer_spc = None
                 __buffer_cspc = None
                 __buffer_dc = None
                 __dataReady = False
                 __timeInterval = None
                 n = None
                 def __init__(self):
                     Operation.__init__(self)
                 def setup(self, n=None, timeInterval=None, overlapping=False):
                     """
                     Set the parameters of the integration class.
                     Inputs:
                         n        :    Number of coherent integrations
                         timeInterval   :    Time of integration. If the parameter "n" is selected this one does not work
                         overlapping    :
                     """
                     self.__initime = None
                     self.__lastdatatime = 0
                     self.__buffer_spc = 0
                     self.__buffer_cspc = 0
                     self.__buffer_dc = 0
                     self.__profIndex = 0
                     self.__dataReady = False
                     self.__byTime = False
                     if n is None and timeInterval is None:
                         raise ValueError("n or timeInterval should be specified ...")
                     if n is not None:
                         self.n = int(n)
                     else:
                         self.__integrationtime = int(timeInterval)
                         self.n = None
                         self.__byTime = True
                 def putData(self, data_spc, data_cspc, data_dc):
                     """
                     Add a profile to the __buffer_spc and increase in one the __profileIndex
                     """
                     self.__buffer_spc += data_spc
                     if data_cspc is None:
                         self.__buffer_cspc = None
                     else:
                         self.__buffer_cspc += data_cspc
                     if data_dc is None:
                         self.__buffer_dc = None
                     else:
                         self.__buffer_dc += data_dc
                     self.__profIndex += 1
                     return
                 def pushData(self):
                     """
                     Return the sum of the last profiles and the profiles used in the sum.
                     Affected:
                     self.__profileIndex
                     """
                     data_spc = self.__buffer_spc
                     data_cspc = self.__buffer_cspc
                     data_dc = self.__buffer_dc
                     n = self.__profIndex
                     self.__buffer_spc = 0
                     self.__buffer_cspc = 0
                     self.__buffer_dc = 0
                     self.__profIndex = 0
                     return data_spc, data_cspc, data_dc, n
                 def byProfiles(self, *args):
                     self.__dataReady = False
                     avgdata_spc = None
                     avgdata_cspc = None
                     avgdata_dc = None
                     self.putData(*args)
                     if self.__profIndex == self.n:
                         avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
                         self.n = n
                         self.__dataReady = True
                     return avgdata_spc, avgdata_cspc, avgdata_dc
                 def byTime(self, datatime, *args):
                     self.__dataReady = False
                     avgdata_spc = None
                     avgdata_cspc = None
                     avgdata_dc = None
                     self.putData(*args)
                     if (datatime - self.__initime) >= self.__integrationtime:
                         avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
                         self.n = n
                         self.__dataReady = True
                     return avgdata_spc, avgdata_cspc, avgdata_dc
                 def integrate(self, datatime, *args):
                     if self.__profIndex == 0:
                         self.__initime = datatime
                     if self.__byTime:
                         avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
                             datatime, *args)
                     else:
                         avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
                     if not self.__dataReady:
                         return None, None, None, None
                     return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
                 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
                     if n == 1:
                         return
                     dataOut.flagNoData = True
                     if not self.isConfig:
                         self.setup(n, timeInterval, overlapping)
                         self.isConfig = True
                     avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
                                                                                         dataOut.data_spc,
                                                                                         dataOut.data_cspc,
                                                                                         dataOut.data_dc)
                     if self.__dataReady:
                         dataOut.data_spc = avgdata_spc
                         dataOut.data_cspc = avgdata_cspc
                         dataOut.data_dc = avgdata_dc
                         dataOut.nIncohInt *= self.n
                         dataOut.utctime = avgdatatime
                         dataOut.flagNoData = False
-                    return dataOut
  No newline at end of file
+                    return dataOut

schainpy/model/proc/jroproc_voltage.py +20 -21

             import sys
             import numpy
             from scipy import interpolate
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jrodata import Voltage
             from schainpy.utils import log
             from time import time
             @MPDecorator
             class VoltageProc(ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Voltage()
                     self.flip = 1
                     self.setupReq = False
                 def run(self):
                     if self.dataIn.type == 'AMISR':
                         self.__updateObjFromAmisrInput()
                     if self.dataIn.type == 'Voltage':
                         self.dataOut.copy(self.dataIn)
                 #         self.dataOut.copy(self.dataIn)
                 def __updateObjFromAmisrInput(self):
                     self.dataOut.timeZone = self.dataIn.timeZone
                     self.dataOut.dstFlag = self.dataIn.dstFlag
                     self.dataOut.errorCount = self.dataIn.errorCount
                     self.dataOut.useLocalTime = self.dataIn.useLocalTime
                     self.dataOut.flagNoData = self.dataIn.flagNoData
                     self.dataOut.data = self.dataIn.data
                     self.dataOut.utctime = self.dataIn.utctime
                     self.dataOut.channelList = self.dataIn.channelList
                     #self.dataOut.timeInterval = self.dataIn.timeInterval
                     self.dataOut.heightList = self.dataIn.heightList
                     self.dataOut.nProfiles = self.dataIn.nProfiles
                     self.dataOut.nCohInt = self.dataIn.nCohInt
                     self.dataOut.ippSeconds = self.dataIn.ippSeconds
                     self.dataOut.frequency = self.dataIn.frequency
                     self.dataOut.azimuth = self.dataIn.azimuth
                     self.dataOut.zenith = self.dataIn.zenith
                     self.dataOut.beam.codeList = self.dataIn.beam.codeList
                     self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
                     self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
                 #
                 #        pass#
                 #
                 #    def init(self):
                 #
                 #
                 #        if self.dataIn.type == 'AMISR':
                 #            self.__updateObjFromAmisrInput()
                 #
                 #        if self.dataIn.type == 'Voltage':
                 #            self.dataOut.copy(self.dataIn)
                 #        # No necesita copiar en cada init() los atributos de dataIn
                 #        # la copia deberia hacerse por cada nuevo bloque de datos
                 def selectChannels(self, channelList):
                     channelIndexList = []
                     for channel in channelList:
                         if channel not in self.dataOut.channelList:
                             raise ValueError("Channel %d is not in %s" %(channel, str(self.dataOut.channelList)))
                         index = self.dataOut.channelList.index(channel)
                         channelIndexList.append(index)
                     self.selectChannelsByIndex(channelIndexList)
                 def selectChannelsByIndex(self, channelIndexList):
                     """
                     Selecciona un bloque de datos en base a canales segun el channelIndexList
                     Input:
                         channelIndexList    :    lista sencilla de canales a seleccionar por ej. [2,3,7]
                     Affected:
                         self.dataOut.data
                         self.dataOut.channelIndexList
                         self.dataOut.nChannels
                         self.dataOut.m_ProcessingHeader.totalSpectra
                         self.dataOut.systemHeaderObj.numChannels
                         self.dataOut.m_ProcessingHeader.blockSize
                     Return:
                         None
                     """
                     for channelIndex in channelIndexList:
                         if channelIndex not in self.dataOut.channelIndexList:
                             print(channelIndexList)
                             raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
                     if self.dataOut.flagDataAsBlock:
                         """
                         Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         data = self.dataOut.data[channelIndexList,:,:]
                     else:
                         data = self.dataOut.data[channelIndexList,:]
                     self.dataOut.data = data
                     # self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
                     self.dataOut.channelList = range(len(channelIndexList))
                     return 1
                 def selectHeights(self, minHei=None, maxHei=None):
                     """
                     Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
                     minHei <= height <= maxHei
                     Input:
                         minHei    :    valor minimo de altura a considerar
                         maxHei    :    valor maximo de altura a considerar
                     Affected:
                         Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if minHei == None:
                         minHei = self.dataOut.heightList[0]
                     if maxHei == None:
                         maxHei = self.dataOut.heightList[-1]
                     if (minHei < self.dataOut.heightList[0]):
                         minHei = self.dataOut.heightList[0]
                     if (maxHei > self.dataOut.heightList[-1]):
                         maxHei = self.dataOut.heightList[-1]
                     minIndex = 0
                     maxIndex = 0
                     heights = self.dataOut.heightList
                     inda = numpy.where(heights >= minHei)
                     indb = numpy.where(heights <= maxHei)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(heights)
                     self.selectHeightsByIndex(minIndex, maxIndex)
                     return 1
                 def selectHeightsByIndex(self, minIndex, maxIndex):
                     """
                     Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
                     minIndex <= index <= maxIndex
                     Input:
                         minIndex    :    valor de indice minimo de altura a considerar
                         maxIndex    :    valor de indice maximo de altura a considerar
                     Affected:
                         self.dataOut.data
                         self.dataOut.heightList
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if (minIndex < 0) or (minIndex > maxIndex):
                         raise ValueError("Height index range (%d,%d) is not valid" % (minIndex, maxIndex))
                     if (maxIndex >= self.dataOut.nHeights):
                         maxIndex = self.dataOut.nHeights
                     #voltage
                     if self.dataOut.flagDataAsBlock:
                         """
                         Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         data = self.dataOut.data[:,:, minIndex:maxIndex]
                     else:
                         data = self.dataOut.data[:, minIndex:maxIndex]
                     #         firstHeight = self.dataOut.heightList[minIndex]
                     self.dataOut.data = data
                     self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
                     if self.dataOut.nHeights <= 1:
                         raise ValueError("selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights))
                     return 1
                 def filterByHeights(self, window):
                     deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     if window == None:
                         window = (self.dataOut.radarControllerHeaderObj.txA/self.dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
                     newdelta = deltaHeight * window
                     r = self.dataOut.nHeights % window
                     newheights = (self.dataOut.nHeights-r)/window
                     if newheights <= 1:
                         raise ValueError("filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(self.dataOut.nHeights, window))
                     if self.dataOut.flagDataAsBlock:
                         """
                         Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         buffer = self.dataOut.data[:, :, 0:int(self.dataOut.nHeights-r)]
                         buffer = buffer.reshape(self.dataOut.nChannels, self.dataOut.nProfiles, int(self.dataOut.nHeights/window), window)
                         buffer = numpy.sum(buffer,3)
                     else:
                         buffer = self.dataOut.data[:,0:int(self.dataOut.nHeights-r)]
                         buffer = buffer.reshape(self.dataOut.nChannels,int(self.dataOut.nHeights/window),int(window))
                         buffer = numpy.sum(buffer,2)
                     self.dataOut.data = buffer
                     self.dataOut.heightList = self.dataOut.heightList[0] + numpy.arange( newheights )*newdelta
                     self.dataOut.windowOfFilter = window
                 def setH0(self, h0, deltaHeight = None):
                     if not deltaHeight:
                         deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     nHeights = self.dataOut.nHeights
                     newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
                     self.dataOut.heightList = newHeiRange
                 def deFlip(self, channelList = []):
                     data = self.dataOut.data.copy()
                     if self.dataOut.flagDataAsBlock:
                         flip = self.flip
                         profileList = list(range(self.dataOut.nProfiles))
                         if not channelList:
                             for thisProfile in profileList:
                                 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
                                 flip *= -1.0
                         else:
                             for thisChannel in channelList:
                                 if thisChannel not in self.dataOut.channelList:
                                     continue
                                 for thisProfile in profileList:
                                     data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
                                     flip *= -1.0
                         self.flip = flip
                     else:
                         if not channelList:
                             data[:,:] = data[:,:]*self.flip
                         else:
                             for thisChannel in channelList:
                                 if thisChannel not in self.dataOut.channelList:
                                     continue
                                 data[thisChannel,:] = data[thisChannel,:]*self.flip
                         self.flip *= -1.
                     self.dataOut.data = data
                 def setRadarFrequency(self, frequency=None):
                     if frequency != None:
                         self.dataOut.frequency = frequency
                     return 1
                 def interpolateHeights(self, topLim, botLim):
                     #69 al 72 para julia
                     #82-84 para meteoros
                     if len(numpy.shape(self.dataOut.data))==2:
                         sampInterp = (self.dataOut.data[:,botLim-1] + self.dataOut.data[:,topLim+1])/2
                         sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
                         #self.dataOut.data[:,botLim:limSup+1] = sampInterp
                         self.dataOut.data[:,botLim:topLim+1] = sampInterp
                     else:
                         nHeights = self.dataOut.data.shape[2]
                         x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
                         y = self.dataOut.data[:,:,list(range(botLim))+list(range(topLim+1,nHeights))]
                         f = interpolate.interp1d(x, y, axis = 2)
                         xnew = numpy.arange(botLim,topLim+1)
                         ynew = f(xnew)
                         self.dataOut.data[:,:,botLim:topLim+1]  = ynew
                 # import collections
             class CohInt(Operation):
                 isConfig = False
                 __profIndex = 0
                 __byTime = False
                 __initime = None
                 __lastdatatime = None
                 __integrationtime = None
                 __buffer = None
                 __bufferStride = []
                 __dataReady = False
                 __profIndexStride = 0
                 __dataToPutStride = False
                 n = None
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     #   self.isConfig = False
                 def setup(self, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False):
                     """
                     Set the parameters of the integration class.
                     Inputs:
                         n               :    Number of coherent integrations
                         timeInterval    :    Time of integration. If the parameter "n" is selected this one does not work
                         overlapping     :
                     """
                     self.__initime = None
                     self.__lastdatatime = 0
                     self.__buffer = None
                     self.__dataReady = False
                     self.byblock = byblock
                     self.stride = stride
                     if n == None and timeInterval == None:
                         raise ValueError("n or timeInterval should be specified ...")
                     if n != None:
                         self.n = n
                         self.__byTime = False
                     else:
                         self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
                         self.n = 9999
                         self.__byTime = True
                     if overlapping:
                         self.__withOverlapping = True
                         self.__buffer = None
                     else:
                         self.__withOverlapping = False
                         self.__buffer = 0
                     self.__profIndex = 0
                 def putData(self, data):
                     """
                     Add a profile to the __buffer and increase in one the __profileIndex
                     """
                     if not self.__withOverlapping:
                         self.__buffer += data.copy()
                         self.__profIndex += 1
                         return
                     #Overlapping data
                     nChannels, nHeis = data.shape
                     data = numpy.reshape(data, (1, nChannels, nHeis))
                     #If the buffer is empty then it takes the data value
                     if self.__buffer is None:
                         self.__buffer = data
                         self.__profIndex += 1
                         return
                     #If the buffer length is lower than n then stakcing the data value
                     if self.__profIndex < self.n:
                         self.__buffer = numpy.vstack((self.__buffer, data))
                         self.__profIndex += 1
                         return
                     #If the buffer length is equal to n then replacing the last buffer value with the data value
                     self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
                     self.__buffer[self.n-1] = data
                     self.__profIndex = self.n
                     return
                 def pushData(self):
                     """
                     Return the sum of the last profiles and the profiles used in the sum.
                     Affected:
                     self.__profileIndex
                     """
                     if not self.__withOverlapping:
                         data = self.__buffer
                         n = self.__profIndex
                         self.__buffer = 0
                         self.__profIndex = 0
                         return data, n
                     #Integration with Overlapping
                     data = numpy.sum(self.__buffer, axis=0)
                     # print data
                     # raise
                     n = self.__profIndex
                     return data, n
                 def byProfiles(self, data):
                     self.__dataReady = False
                     avgdata = None
                     #         n = None
                     # print data
                     # raise
                     self.putData(data)
                     if self.__profIndex == self.n:
                         avgdata, n = self.pushData()
                         self.__dataReady = True
                     return avgdata
                 def byTime(self, data, datatime):
                     self.__dataReady = False
                     avgdata = None
                     n = None
                     self.putData(data)
                     if (datatime - self.__initime) >= self.__integrationtime:
                         avgdata, n = self.pushData()
                         self.n = n
                         self.__dataReady = True
                     return avgdata
                 def integrateByStride(self, data, datatime):
                     # print data
                     if self.__profIndex == 0:
                         self.__buffer = [[data.copy(), datatime]]
                     else:
                         self.__buffer.append([data.copy(),datatime])
                     self.__profIndex += 1
                     self.__dataReady = False
                     if self.__profIndex == self.n * self.stride :
                         self.__dataToPutStride = True
                         self.__profIndexStride = 0
                         self.__profIndex = 0
                         self.__bufferStride = []
                         for i in range(self.stride):
                             current = self.__buffer[i::self.stride]
                             data = numpy.sum([t[0] for t in current], axis=0)
                             avgdatatime = numpy.average([t[1] for t in current])
                             # print data
                             self.__bufferStride.append((data, avgdatatime))
                     if self.__dataToPutStride:
                         self.__dataReady = True
                         self.__profIndexStride += 1
                         if self.__profIndexStride == self.stride:
                             self.__dataToPutStride = False
                         # print self.__bufferStride[self.__profIndexStride - 1]
                         # raise
                         return self.__bufferStride[self.__profIndexStride - 1]
                     return None, None
                 def integrate(self, data, datatime=None):
                     if self.__initime == None:
                         self.__initime = datatime
                     if self.__byTime:
                         avgdata = self.byTime(data, datatime)
                     else:
                         avgdata = self.byProfiles(data)
                     self.__lastdatatime = datatime
                     if avgdata is None:
                         return None, None
                     avgdatatime = self.__initime
                     deltatime = datatime - self.__lastdatatime
                     if not self.__withOverlapping:
                         self.__initime = datatime
                     else:
                         self.__initime += deltatime
                     return avgdata, avgdatatime
                 def integrateByBlock(self, dataOut):
                     times = int(dataOut.data.shape[1]/self.n)
                     avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
                     id_min = 0
                     id_max = self.n
                     for i in range(times):
                         junk = dataOut.data[:,id_min:id_max,:]
                         avgdata[:,i,:] = junk.sum(axis=1)
                         id_min += self.n
                         id_max += self.n
                     timeInterval = dataOut.ippSeconds*self.n
                     avgdatatime = (times - 1) * timeInterval + dataOut.utctime
                     self.__dataReady = True
                     return avgdata, avgdatatime
                 def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
                     if not self.isConfig:
                         self.setup(n=n, stride=stride, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
                         self.isConfig = True
                     if dataOut.flagDataAsBlock:
                         """
                         Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         avgdata, avgdatatime = self.integrateByBlock(dataOut)
                         dataOut.nProfiles /= self.n
                     else:
                         if stride is None:
                             avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
                         else:
                             avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
                     #   dataOut.timeInterval *= n
                     dataOut.flagNoData = True
                     if self.__dataReady:
                         dataOut.data = avgdata
                         dataOut.nCohInt *= self.n
                         dataOut.utctime = avgdatatime
                         # print avgdata, avgdatatime
                         # raise
                         #   dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
                         dataOut.flagNoData = False
                     return dataOut
             class Decoder(Operation):
                 isConfig = False
                 __profIndex = 0
                 code = None
                 nCode = None
                 nBaud = None
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.times = None
                     self.osamp = None
                 #         self.__setValues = False
                     self.isConfig = False
                     self.setupReq = False
                 def setup(self, code, osamp, dataOut):
                     self.__profIndex = 0
                     self.code = code
                     self.nCode = len(code)
                     self.nBaud = len(code[0])
                     if (osamp != None) and (osamp >1):
                         self.osamp = osamp
                         self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
                         self.nBaud = self.nBaud*self.osamp
                     self.__nChannels = dataOut.nChannels
                     self.__nProfiles = dataOut.nProfiles
                     self.__nHeis = dataOut.nHeights
                     if self.__nHeis < self.nBaud:
                         raise ValueError('Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud))
                     #Frequency
                     __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
                     __codeBuffer[:,0:self.nBaud] = self.code
                     self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
                     if dataOut.flagDataAsBlock:
                         self.ndatadec = self.__nHeis #- self.nBaud + 1
                         self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
                     else:
                         #Time
                         self.ndatadec = self.__nHeis #- self.nBaud + 1
                         self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
                 def __convolutionInFreq(self, data):
                     fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
                     fft_data = numpy.fft.fft(data, axis=1)
                     conv = fft_data*fft_code
                     data = numpy.fft.ifft(conv,axis=1)
                     return data
                 def __convolutionInFreqOpt(self, data):
                     raise NotImplementedError
                 def __convolutionInTime(self, data):
                     code = self.code[self.__profIndex]
                     for i in range(self.__nChannels):
                         self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
                     return self.datadecTime
                 def __convolutionByBlockInTime(self, data):
                     repetitions = int(self.__nProfiles / self.nCode)
                     junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
                     junk = junk.flatten()
                     code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
                     profilesList = range(self.__nProfiles)
                     for i in range(self.__nChannels):
                         for j in profilesList:
                             self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
                     return self.datadecTime
                 def __convolutionByBlockInFreq(self, data):
                     raise NotImplementedError("Decoder by frequency fro Blocks not implemented")
                     fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
                     fft_data = numpy.fft.fft(data, axis=2)
                     conv = fft_data*fft_code
                     data = numpy.fft.ifft(conv,axis=2)
                     return data
                 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
                     if dataOut.flagDecodeData:
                         print("This data is already decoded, recoding again ...")
                     if not self.isConfig:
                         if code is None:
                             if dataOut.code is None:
                                 raise ValueError("Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type)
                             code = dataOut.code
                         else:
                             code = numpy.array(code).reshape(nCode,nBaud)
                         self.setup(code, osamp, dataOut)
                         self.isConfig = True
                         if mode == 3:
                             sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
                         if times != None:
                             sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
                     if self.code is None:
                         print("Fail decoding: Code is not defined.")
                         return
                     self.__nProfiles = dataOut.nProfiles
                     datadec = None
                     if mode == 3:
                         mode = 0
                     if dataOut.flagDataAsBlock:
                         """
                         Decoding when data have been read as block,
                         """
                         if mode == 0:
                             datadec = self.__convolutionByBlockInTime(dataOut.data)
                         if mode == 1:
                             datadec = self.__convolutionByBlockInFreq(dataOut.data)
                     else:
                         """
                         Decoding when data have been read profile by profile
                         """
                         if mode == 0:
                             datadec = self.__convolutionInTime(dataOut.data)
                         if mode == 1:
                             datadec = self.__convolutionInFreq(dataOut.data)
                         if mode == 2:
                             datadec = self.__convolutionInFreqOpt(dataOut.data)
                     if datadec is None:
                         raise ValueError("Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode)
                     dataOut.code = self.code
                     dataOut.nCode = self.nCode
                     dataOut.nBaud = self.nBaud
                     dataOut.data = datadec
                     dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
                     dataOut.flagDecodeData = True #asumo q la data esta decodificada
                     if self.__profIndex == self.nCode-1:
                         self.__profIndex = 0
                         return dataOut
                     self.__profIndex += 1
                     return dataOut
                 #        dataOut.flagDeflipData = True #asumo q la data no esta sin flip
             class ProfileConcat(Operation):
                 isConfig = False
                 buffer = None
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.profileIndex = 0
                 def reset(self):
                     self.buffer = numpy.zeros_like(self.buffer)
                     self.start_index = 0
                     self.times = 1
                 def setup(self, data, m, n=1):
                     self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
                     self.nHeights = data.shape[1]#.nHeights
                     self.start_index = 0
                     self.times = 1
                 def concat(self, data):
                     self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
                     self.start_index = self.start_index + self.nHeights
                 def run(self, dataOut, m):
                     dataOut.flagNoData = True
                     if not self.isConfig:
                         self.setup(dataOut.data, m, 1)
                         self.isConfig = True
                     if dataOut.flagDataAsBlock:
                         raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
                     else:
                         self.concat(dataOut.data)
                         self.times += 1
                         if self.times > m:
                             dataOut.data = self.buffer
                             self.reset()
                             dataOut.flagNoData = False
                             # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
                             deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                             xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
                             dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
                             dataOut.ippSeconds *= m
                     return dataOut
             class ProfileSelector(Operation):
                 profileIndex = None
                 # Tamanho total de los perfiles
                 nProfiles = None
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.profileIndex = 0
                 def incProfileIndex(self):
                     self.profileIndex += 1
                     if self.profileIndex >= self.nProfiles:
                         self.profileIndex = 0
                 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
                     if profileIndex < minIndex:
                         return False
                     if profileIndex > maxIndex:
                         return False
                     return True
                 def isThisProfileInList(self, profileIndex, profileList):
                     if profileIndex not in profileList:
                         return False
                     return True
                 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
                     """
                     ProfileSelector:
                     Inputs:
                         profileList        :    Index of profiles selected. Example: profileList = (0,1,2,7,8)
                         profileRangeList    :    Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
                         rangeList            :    List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
                     """
                     if rangeList is not None:
                         if type(rangeList[0]) not in (tuple, list):
                             rangeList = [rangeList]
                     dataOut.flagNoData = True
                     if dataOut.flagDataAsBlock:
                         """
                         data dimension  = [nChannels, nProfiles, nHeis]
                         """
                         if profileList != None:
                             dataOut.data = dataOut.data[:,profileList,:]
                         if profileRangeList != None:
                             minIndex = profileRangeList[0]
                             maxIndex = profileRangeList[1]
                             profileList = list(range(minIndex, maxIndex+1))
                             dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
                         if rangeList != None:
                             profileList = []
                             for thisRange in rangeList:
                                 minIndex = thisRange[0]
                                 maxIndex = thisRange[1]
                                 profileList.extend(list(range(minIndex, maxIndex+1)))
                             dataOut.data = dataOut.data[:,profileList,:]
                         dataOut.nProfiles = len(profileList)
                         dataOut.profileIndex = dataOut.nProfiles - 1
                         dataOut.flagNoData = False
                         return dataOut
                     """
                     data dimension  = [nChannels, nHeis]
                     """
                     if profileList != None:
                         if self.isThisProfileInList(dataOut.profileIndex, profileList):
                             self.nProfiles = len(profileList)
                             dataOut.nProfiles = self.nProfiles
                             dataOut.profileIndex = self.profileIndex
                             dataOut.flagNoData = False
                             self.incProfileIndex()
                         return dataOut
                     if profileRangeList != None:
                         minIndex = profileRangeList[0]
                         maxIndex = profileRangeList[1]
                         if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
                             self.nProfiles = maxIndex - minIndex + 1
                             dataOut.nProfiles = self.nProfiles
                             dataOut.profileIndex = self.profileIndex
                             dataOut.flagNoData = False
                             self.incProfileIndex()
                         return dataOut
                     if rangeList != None:
                         nProfiles = 0
                         for thisRange in rangeList:
                             minIndex = thisRange[0]
                             maxIndex = thisRange[1]
                             nProfiles += maxIndex - minIndex + 1
                         for thisRange in rangeList:
                             minIndex = thisRange[0]
                             maxIndex = thisRange[1]
                             if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
                                 self.nProfiles = nProfiles
                                 dataOut.nProfiles = self.nProfiles
                                 dataOut.profileIndex = self.profileIndex
                                 dataOut.flagNoData = False
                                 self.incProfileIndex()
                                 break
                         return dataOut
                     if beam != None: #beam is only for AMISR data
                         if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
                             dataOut.flagNoData = False
                             dataOut.profileIndex = self.profileIndex
                             self.incProfileIndex()
                         return dataOut
                     raise ValueError("ProfileSelector needs profileList, profileRangeList or rangeList parameter")
                     #return False
                     return dataOut
             class Reshaper(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.__buffer = None
                     self.__nitems = 0
                 def __appendProfile(self, dataOut, nTxs):
                     if self.__buffer is None:
                         shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
                         self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
                     ini = dataOut.nHeights * self.__nitems
                     end = ini + dataOut.nHeights
                     self.__buffer[:, ini:end] = dataOut.data
                     self.__nitems += 1
                     return int(self.__nitems*nTxs)
                 def __getBuffer(self):
                     if self.__nitems == int(1./self.__nTxs):
                         self.__nitems = 0
                         return self.__buffer.copy()
                     return None
                 def __checkInputs(self, dataOut, shape, nTxs):
                     if shape is None and nTxs is None:
                         raise ValueError("Reshaper: shape of factor should be defined")
                     if nTxs:
                         if nTxs < 0:
                             raise ValueError("nTxs should be greater than 0")
                         if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
                             raise ValueError("nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs)))
                         shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
                         return shape, nTxs
                     if len(shape) != 2 and len(shape) !=  3:
                         raise ValueError("shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights))
                     if len(shape) == 2:
                         shape_tuple = [dataOut.nChannels]
                         shape_tuple.extend(shape)
                     else:
                         shape_tuple = list(shape)
                     nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
                     return shape_tuple, nTxs
                 def run(self, dataOut, shape=None, nTxs=None):
                     shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
                     dataOut.flagNoData = True
                     profileIndex = None
                     if dataOut.flagDataAsBlock:
                         dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
                         dataOut.flagNoData = False
                         profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
                     else:
                         if self.__nTxs < 1:
                             self.__appendProfile(dataOut, self.__nTxs)
                             new_data = self.__getBuffer()
                             if new_data is not None:
                                 dataOut.data = new_data
                                 dataOut.flagNoData = False
                                 profileIndex = dataOut.profileIndex*nTxs
                         else:
                             raise ValueError("nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)")
                     deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                     dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
                     dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
                     dataOut.profileIndex = profileIndex
                     dataOut.ippSeconds /= self.__nTxs
                     return dataOut
             class SplitProfiles(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                 def run(self, dataOut, n):
                     dataOut.flagNoData = True
                     profileIndex = None
                     if dataOut.flagDataAsBlock:
                         #nchannels, nprofiles, nsamples
                         shape = dataOut.data.shape
                         if shape[2] % n != 0:
                             raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
                         new_shape = shape[0], shape[1]*n, int(shape[2]/n)
                         dataOut.data = numpy.reshape(dataOut.data, new_shape)
                         dataOut.flagNoData = False
                         profileIndex = int(dataOut.nProfiles/n) - 1
                     else:
                         raise ValueError("Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)")
                     deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                     dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
                     dataOut.nProfiles = int(dataOut.nProfiles*n)
                     dataOut.profileIndex = profileIndex
                     dataOut.ippSeconds /= n
                     return dataOut
             class CombineProfiles(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.__remData = None
                     self.__profileIndex = 0
                 def run(self, dataOut, n):
                     dataOut.flagNoData = True
                     profileIndex = None
                     if dataOut.flagDataAsBlock:
                         #nchannels, nprofiles, nsamples
                         shape = dataOut.data.shape
                         new_shape = shape[0], shape[1]/n, shape[2]*n
                         if shape[1] % n != 0:
                             raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[1]))
                         dataOut.data = numpy.reshape(dataOut.data, new_shape)
                         dataOut.flagNoData = False
                         profileIndex = int(dataOut.nProfiles*n) - 1
                     else:
                         #nchannels, nsamples
                         if self.__remData is None:
                             newData = dataOut.data
                         else:
                             newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
                         self.__profileIndex += 1
                         if self.__profileIndex < n:
                             self.__remData = newData
                             #continue
                             return
                         self.__profileIndex = 0
                         self.__remData = None
                         dataOut.data = newData
                         dataOut.flagNoData = False
                         profileIndex = dataOut.profileIndex/n
                     deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                     dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
                     dataOut.nProfiles = int(dataOut.nProfiles/n)
                     dataOut.profileIndex = profileIndex
                     dataOut.ippSeconds *= n
                     return dataOut
             # import collections
             # from scipy.stats import mode
             #
             # class Synchronize(Operation):
             #
             #     isConfig = False
             #     __profIndex = 0
             #
             #     def __init__(self, **kwargs):
             #
             #         Operation.__init__(self, **kwargs)
             # #         self.isConfig = False
             #         self.__powBuffer = None
             #         self.__startIndex = 0
             #         self.__pulseFound = False
             #
             #     def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
             #
             #         #Read data
             #
             #         powerdB = dataOut.getPower(channel = channel)
             #         noisedB = dataOut.getNoise(channel = channel)[0]
             #
             #         self.__powBuffer.extend(powerdB.flatten())
             #
             #         dataArray = numpy.array(self.__powBuffer)
             #
             #         filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
             #
             #         maxValue = numpy.nanmax(filteredPower)
             #
             #         if maxValue < noisedB + 10:
             #             #No se encuentra ningun pulso de transmision
             #             return None
             #
             #         maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
             #
             #         if len(maxValuesIndex) < 2:
             #             #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
             #             return None
             #
             #         phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
             #
             #         #Seleccionar solo valores con un espaciamiento de nSamples
             #         pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
             #
             #         if len(pulseIndex) < 2:
             #             #Solo se encontro un pulso de transmision con ancho mayor a 1
             #             return None
             #
             #         spacing = pulseIndex[1:] - pulseIndex[:-1]
             #
             #         #remover senales que se distancien menos de 10 unidades o muestras
             #         #(No deberian existir IPP menor a 10 unidades)
             #
             #         realIndex = numpy.where(spacing > 10 )[0]
             #
             #         if len(realIndex) < 2:
             #             #Solo se encontro un pulso de transmision con ancho mayor a 1
             #             return None
             #
             #         #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
             #         realPulseIndex = pulseIndex[realIndex]
             #
             #         period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
             #
             #         print "IPP = %d samples" %period
             #
             #         self.__newNSamples = dataOut.nHeights #int(period)
             #         self.__startIndex = int(realPulseIndex[0])
             #
             #         return 1
             #
             #
             #     def setup(self, nSamples, nChannels, buffer_size = 4):
             #
             #         self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
             #                                           maxlen = buffer_size*nSamples)
             #
             #         bufferList = []
             #
             #         for i in range(nChannels):
             #             bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) +  numpy.NAN,
             #                                           maxlen = buffer_size*nSamples)
             #
             #             bufferList.append(bufferByChannel)
             #
             #         self.__nSamples = nSamples
             #         self.__nChannels = nChannels
             #         self.__bufferList = bufferList
             #
             #     def run(self, dataOut, channel = 0):
             #
             #         if not self.isConfig:
             #             nSamples = dataOut.nHeights
             #             nChannels = dataOut.nChannels
             #             self.setup(nSamples, nChannels)
             #             self.isConfig = True
             #
             #         #Append new data to internal buffer
             #         for thisChannel in range(self.__nChannels):
             #             bufferByChannel = self.__bufferList[thisChannel]
             #             bufferByChannel.extend(dataOut.data[thisChannel])
             #
             #         if self.__pulseFound:
             #             self.__startIndex -= self.__nSamples
             #
             #         #Finding Tx Pulse
             #         if not self.__pulseFound:
             #             indexFound = self.__findTxPulse(dataOut, channel)
             #
             #             if indexFound == None:
             #                 dataOut.flagNoData = True
             #                 return
             #
             #             self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
             #             self.__pulseFound = True
             #             self.__startIndex = indexFound
             #
             #         #If pulse was found ...
             #         for thisChannel in range(self.__nChannels):
             #             bufferByChannel = self.__bufferList[thisChannel]
             #             #print self.__startIndex
             #             x = numpy.array(bufferByChannel)
             #             self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
             #
             #         deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
             #         dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
             # #             dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
             #
             #         dataOut.data = self.__arrayBuffer
             #
             #         self.__startIndex += self.__newNSamples
             #
             #         return

schainpy/model/utils/jroutils_ftp.py +27 -27

             '''
             @author: Daniel Suarez
             '''
             import os
             import glob
             import ftplib
             try:
                 import paramiko
                 import scp
             except:
                 print("You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server")
             import time
             import threading
             Thread = threading.Thread
             # try:
             #     from gevent import sleep
             # except:
             from time import sleep
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             class Remote(Thread):
                 """
                 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
                 used to upload or download files remotely.
                 Non-standard Python modules used:
                     None
                 Written by:
                     "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
                 """
                 server = None
                 username = None
                 password = None
                 remotefolder = None
                 period = 60
                 fileList = []
                 bussy = False
                 def __init__(self, server, username, password, remotefolder, period=60):
                     Thread.__init__(self)
                     self.setDaemon(True)
                     self.status = 0
                     self.__server = server
                     self.__username = username
                     self.__password = password
                     self.__remotefolder = remotefolder
                     self.period = period
                     self.fileList = []
                     self.bussy = False
                     self.stopFlag = False
                     print("[Remote Server] Opening server: %s" %self.__server)
                     if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
                         print("[Remote Server] %s server was opened successfully" %self.__server)
                     self.close()
                     self.mutex = threading.Lock()
                 def stop(self):
                     self.stopFlag = True
                     self.join(10)
                 def open(self):
                     """
                     Connect to server and create a connection class (FTP or SSH) to remote server.
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def close(self):
                     """
                     Close connection to server
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def mkdir(self, remotefolder):
                     """
                     Create a folder remotely
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def cd(self, remotefolder):
                     """
                     Change working directory in remote server
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def download(self, filename, localfolder=None):
                     """
                     Download a file from server to local host
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def sendFile(self, fullfilename):
                     """
                     sendFile method is used to upload a local file to the current directory in remote server
                     Inputs:
                         fullfilename        - full path name of local file to store in remote directory
                     Returns:
 in error case else 1
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def upload(self, fullfilename, remotefolder=None):
                     """
                     upload method is used to upload a local file to remote directory. This method changes
                     working directory before sending a file.
                     Inputs:
                         fullfilename        - full path name of local file to store in remote directory
                         remotefolder    - remote directory
                     Returns:
 in error case else 1
                     """
                     print("[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder))
                     if not self.status:
                         return 0
                     if remotefolder == None:
                         remotefolder = self.remotefolder
                     if not self.cd(remotefolder):
                         return 0
                     if not self.sendFile(fullfilename):
                         print("[Remote Server] Error uploading file %s" %fullfilename)
                         return 0
                     print("[Remote Server] upload finished successfully")
                     return 1
                 def delete(self, filename):
                     """
                     Remove a file from remote server
                     """
                     pass
                 def updateFileList(self, fileList):
                     """
                     Remove a file from remote server
                     """
                     if fileList == self.fileList:
                         return 0
                     self.mutex.acquire()
                     #         init = time.time()
+                    #
                     #         while(self.bussy):
                     #             sleep(0.1)
                     #             if time.time() - init > 2*self.period:
                     #                 return 0
                     self.fileList = fileList
                     self.mutex.release()
                     return 1
                 def run(self):
                     if not self.status:
                         print("Finishing FTP service")
                         return
                     if not self.cd(self.remotefolder):
                         raise ValueError("Could not access to the new remote directory: %s" %self.remotefolder)
                     while True:
                         for i in range(self.period):
                             if self.stopFlag:
                                 break
                             sleep(1)
                         if self.stopFlag:
                             break
                         #   self.bussy = True
                         self.mutex.acquire()
                         print("[Remote Server] Opening %s" %self.__server)
                         if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
                             self.mutex.release()
                             continue
                         for thisFile in self.fileList:
                             self.upload(thisFile, self.remotefolder)
                         print("[Remote Server] Closing %s" %self.__server)
                         self.close()
                         self.mutex.release()
                         # self.bussy = False
                     print("[Remote Server] Thread stopped successfully")
             class FTPClient(Remote):
                 __ftpClientObj = None
                 def __init__(self, server, username, password, remotefolder, period=60):
                     """
                     """
                     Remote.__init__(self, server, username, password, remotefolder, period)
                 def open(self, server, username, password, remotefolder):
                     """
                     This method is used to set FTP parameters and establish a connection to remote server
                     Inputs:
                         server    - remote server IP Address
                         username    - remote server Username
                         password    - remote server password
                         remotefolder    - remote server current working directory
                     Return:
                         Boolean     -     Returns 1 if a connection has been established, 0 otherwise
                     Affects:
                         self.status        - in case of error or fail connection this parameter is set to 0 else 1
                     """
                     if server == None:
                         raise ValueError("FTP server should be defined")
                     if username == None:
                         raise ValueError("FTP username should be defined")
                     if password == None:
                         raise ValueError("FTP password should be defined")
                     if remotefolder == None:
                         raise ValueError("FTP remote folder should be defined")
                     try:
                         ftpClientObj = ftplib.FTP(server)
                     except ftplib.all_errors as e:
                         print("[FTP Server]: FTP server connection fail: %s" %server)
                         print("[FTP Server]:", e)
                         self.status = 0
                         return 0
                     try:
                         ftpClientObj.login(username, password)
                     except ftplib.all_errors:
                         print("[FTP Server]: FTP username or password are incorrect")
                         self.status = 0
                         return 0
                     if remotefolder == None:
                         remotefolder = ftpClientObj.pwd()
                     else:
                         try:
                             ftpClientObj.cwd(remotefolder)
                         except ftplib.all_errors:
                             print("[FTP Server]: FTP remote folder is invalid: %s" %remotefolder)
                             remotefolder = ftpClientObj.pwd()
                     self.server = server
                     self.username = username
                     self.password = password
                     self.remotefolder = remotefolder
                     self.__ftpClientObj = ftpClientObj
                     self.status = 1
                     return 1
                 def close(self):
                     """
                     Close connection to remote server
                     """
                     if not self.status:
                         return 0
                     self.__ftpClientObj.close()
                 def mkdir(self, remotefolder):
                     """
                     mkdir is used to make a new directory in remote server
                     Input:
                         remotefolder    - directory name
                     Return:
 in error case else 1
                     """
                     if not self.status:
                         return 0
                     try:
                         self.__ftpClientObj.mkd(dirname)
                     except ftplib.all_errors:
                         print("[FTP Server]: Error creating remote folder: %s" %remotefolder)
                         return 0
                     return 1
                 def cd(self, remotefolder):
                     """
                     cd is used to change remote working directory on server
                     Input:
                         remotefolder    - current working directory
                     Affects:
                         self.remotefolder
                     Return:
 in case of error else 1
                     """
                     if not self.status:
                         return 0
                     if remotefolder == self.remotefolder:
                         return 1
                     try:
                         self.__ftpClientObj.cwd(remotefolder)
                     except ftplib.all_errors:
                         print('[FTP Server]: Error changing to %s' %remotefolder)
                         print('[FTP Server]: Trying to create remote folder')
                         if not self.mkdir(remotefolder):
                             print('[FTP Server]: Remote folder could not be created')
                             return 0
                         try:
                             self.__ftpClientObj.cwd(remotefolder)
                         except ftplib.all_errors:
                             return 0
                     self.remotefolder = remotefolder
                     return 1
                 def sendFile(self, fullfilename):
                     if not self.status:
                         return 0
                     fp = open(fullfilename, 'rb')
                     filename = os.path.basename(fullfilename)
                     command = "STOR %s" %filename
                     try:
                         self.__ftpClientObj.storbinary(command, fp)
                     except ftplib.all_errors as e:
                         print("[FTP Server]:", e)
                         return 0
                     try:
                         self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
                     except ftplib.all_errors as e:
                         print("[FTP Server]:", e)
                     fp.close()
                     return 1
             class SSHClient(Remote):
                 __sshClientObj = None
                 __scpClientObj = None
                 def __init__(self, server, username, password, remotefolder, period=60):
                     """
                     """
                     Remote.__init__(self, server, username, password, remotefolder, period)
                 def open(self, server, username, password, remotefolder, port=22):
                     """
                         This method is used to set SSH parameters and establish a connection to a remote server
                         Inputs:
                             server    - remote server IP Address
                             username    - remote server Username
                             password    - remote server password
                             remotefolder    - remote server current working directory
                         Return: void
                         Affects:
                             self.status        - in case of error or fail connection this parameter is set to 0 else 1
                     """
                     import socket
                     if server == None:
                         raise ValueError("SSH server should be defined")
                     if username == None:
                         raise ValueError("SSH username should be defined")
                     if password == None:
                         raise ValueError("SSH password should be defined")
                     if remotefolder == None:
                         raise ValueError("SSH remote folder should be defined")
                     sshClientObj = paramiko.SSHClient()
                     sshClientObj.load_system_host_keys()
                     sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
                     self.status = 0
                     try:
                         sshClientObj.connect(server, username=username, password=password, port=port)
                     except paramiko.AuthenticationException as e:
                 #             print "SSH username or password are incorrect: %s"
                         print("[SSH Server]:", e)
                         return 0
                     except SSHException as e:
                         print("[SSH Server]:", e)
                         return 0
                     except socket.error:
                         self.status = 0
                         print("[SSH Server]:", e)
                         return 0
                     self.status = 1
                     scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
                     if remotefolder == None:
                         remotefolder = self.pwd()
                     self.server = server
                     self.username = username
                     self.password = password
                     self.__sshClientObj = sshClientObj
                     self.__scpClientObj = scpClientObj
                     self.status = 1
                     if not self.cd(remotefolder):
                         raise ValueError("[SSH Server]: Could not access to remote folder: %s" %remotefolder)
                         return 0
                     self.remotefolder = remotefolder
                     return 1
                 def close(self):
                     """
                         Close connection to remote server
                     """
                     if not self.status:
                         return 0
                     self.__scpClientObj.close()
                     self.__sshClientObj.close()
                 def __execute(self, command):
                     """
                         __execute a command on remote server
                         Input:
                             command    - Exmaple 'ls -l'
                         Return:
 in error case else 1
                     """
                     if not self.status:
                         return 0
                     stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
                     result = stderr.readlines()
                     if len(result) > 1:
                         return 0
                     result = stdout.readlines()
                     if len(result) > 1:
                         return result[0][:-1]
                     return 1
                 def mkdir(self, remotefolder):
                     """
                         mkdir is used to make a new directory in remote server
                         Input:
                             remotefolder    - directory name
                         Return:
 in error case else 1
                     """
                     command = 'mkdir %s' %remotefolder
                     return self.__execute(command)
                 def pwd(self):
                     command = 'pwd'
                     return self.__execute(command)
                 def cd(self, remotefolder):
                     """
                         cd is used to change remote working directory on server
                         Input:
                             remotefolder    - current working directory
                         Affects:
                             self.remotefolder
                         Return:
 in case of error else 1
                     """
                     if not self.status:
                         return 0
                     if remotefolder == self.remotefolder:
                         return 1
                     chk_command = "cd %s; pwd" %remotefolder
                     mkdir_command = "mkdir %s" %remotefolder
                     if not self.__execute(chk_command):
                         if not self.__execute(mkdir_command):
                             self.remotefolder = None
                             return 0
                     self.remotefolder = remotefolder
                     return 1
                 def sendFile(self, fullfilename):
                     if not self.status:
                         return 0
                     try:
                         self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
                     except scp.ScpError as e:
                         print("[SSH Server]", str(e))
                         return 0
                     remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
                     command = 'chmod 775 %s' %remotefile
                     return self.__execute(command)
             class SendToServer(ProcessingUnit):
                 def __init__(self, **kwargs):
                     ProcessingUnit.__init__(self, **kwargs)
                     self.isConfig = False
                     self.clientObj = None
                 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
                     self.clientObj = None
                     self.localfolder = localfolder
                     self.ext = ext
                     self.period = period
                     if str.lower(protocol) == 'ftp':
                         self.clientObj = FTPClient(server, username, password, remotefolder, period)
                     if str.lower(protocol) == 'ssh':
                         self.clientObj = SSHClient(server, username, password, remotefolder, period)
                     if not self.clientObj:
                         raise ValueError("%s has been chosen as remote access protocol but it is not valid" %protocol)
                     self.clientObj.start()
                 def findFiles(self):
                     if not type(self.localfolder) == list:
                         folderList = [self.localfolder]
                     else:
                         folderList = self.localfolder
                     #Remove duplicate items
                     folderList = list(set(folderList))
                     fullfilenameList = []
                     for thisFolder in folderList:
                         print("[Remote Server]: Searching files on %s" %thisFolder)
                         filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
                         if len(filenameList) < 1:
                             continue
                         for thisFile in filenameList:
                             fullfilename = os.path.join(thisFolder, thisFile)
                             if fullfilename in fullfilenameList:
                                 continue
                             #Only files modified in the last 30 minutes are considered
                             if os.path.getmtime(fullfilename) < time.time() - 30*60:
                                 continue
                             fullfilenameList.append(fullfilename)
                     return fullfilenameList
                 def run(self, **kwargs):
                     if not self.isConfig:
                         self.init = time.time()
                         self.setup(**kwargs)
                         self.isConfig = True
                     if not self.clientObj.is_alive():
                         print("[Remote Server]: Restarting connection ")
                         self.setup(**kwargs)
                     if time.time() - self.init >= self.period:
                         fullfilenameList = self.findFiles()
                         if self.clientObj.updateFileList(fullfilenameList):
                             print("[Remote Server]: Sending the next files ", str(fullfilenameList))
                         self.init = time.time()
                 def close(self):
                     print("[Remote Server] Stopping thread")
                     self.clientObj.stop()
             class FTP(object):
                 """
                 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
                 Non-standard Python modules used: None
                 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe  Oct. 26, 2010
                 """
                 def __init__(self,server = None, username=None, password=None, remotefolder=None):
                     """
                     This method is used to setting parameters for FTP and establishing connection to remote server
                     Inputs:
                         server    - remote server IP Address
                         username    - remote server Username
                         password    - remote server password
                         remotefolder    - remote server current working directory
                     Return: void
                     Affects:
                         self.status    - in Error Case or Connection Failed this parameter is set to 1 else 0
                         self.folderList    - sub-folder list of remote folder
                         self.fileList    - file list of remote folder
                     """
                     if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
                         server, username, password, remotefolder = self.parmsByDefault()
                     self.server = server
                     self.username = username
                     self.password = password
                     self.remotefolder = remotefolder
                     self.file = None
                     self.ftp = None
                     self.status = 0
                     try:
                         self.ftp = ftplib.FTP(self.server)
                         self.ftp.login(self.username,self.password)
                         self.ftp.cwd(self.remotefolder)
                         #   print 'Connect to FTP Server: Successfully'
                     except ftplib.all_errors:
                         print('Error FTP Service')
                         self.status = 1
                         return
                     self.dirList = []
                     try:
                         self.dirList = self.ftp.nlst()
                     except ftplib.error_perm as resp:
                         if str(resp) == "550 No files found":
                                 print("no files in this directory")
                                 self.status = 1
                                 return
                     except ftplib.all_errors:
                         print('Error Displaying Dir-Files')
                         self.status = 1
                         return
                     self.fileList = []
                     self.folderList = []
                     #only for test
                     for f in self.dirList:
                         name, ext = os.path.splitext(f)
                         if ext != '':
                             self.fileList.append(f)
                 #                print 'filename: %s - size: %d'%(f,self.ftp.size(f))
                 def parmsByDefault(self):
                     server = 'jro-app.igp.gob.pe'
                     username = 'wmaster'
                     password = 'mst2010vhf'
                     remotefolder = '/home/wmaster/graficos'
                     return server, username, password, remotefolder
                 def mkd(self,dirname):
                     """
                     mkd is used to make directory in remote server
                     Input:
                         dirname    - directory name
                     Return:
 in error case else 0
                     """
                     try:
                         self.ftp.mkd(dirname)
                     except:
                         print('Error creating remote folder:%s'%dirname)
                         return 1
                     return 0
                 def delete(self,filename):
                     """
                     delete is used to delete file in current working directory of remote server
                     Input:
                         filename    - filename to delete in remote folder
                     Return:
 in error case else 0
                     """
                     try:
                         self.ftp.delete(filename)
                     except:
                         print('Error deleting remote file:%s'%filename)
                         return 1
                     return 0
                 def download(self,filename,localfolder):
                     """
                     download is used to downloading file from remote folder into local folder
                     Inputs:
                         filename    - filename to donwload
                         localfolder    - directory local to store filename
                     Returns:
                         self.status    - 1 in error case else 0
                     """
                     self.status = 0
                     if not(filename in self.fileList):
                         print('filename:%s not exists'%filename)
                         self.status = 1
                         return self.status
                     newfilename = os.path.join(localfolder,filename)
                     self.file = open(newfilename, 'wb')
                     try:
                         print('Download: ' + filename)
                         self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
                         print('Download Complete')
                     except ftplib.all_errors:
                         print('Error Downloading ' + filename)
                         self.status = 1
                         return self.status
                     self.file.close()
                     return self.status
                 def __handleDownload(self,block):
                     """
                     __handleDownload is used to handle writing file
                     """
                     self.file.write(block)
                 def upload(self,filename,remotefolder=None):
                     """
                     upload is used to uploading local file to remote directory
                     Inputs:
                         filename    - full path name of local file to store in remote directory
                         remotefolder    - remote directory
                     Returns:
                         self.status    - 1 in error case else 0
                     """
                     if remotefolder == None:
                         remotefolder = self.remotefolder
                     self.status = 0
                     try:
                         self.ftp.cwd(remotefolder)
                         self.file = open(filename, 'rb')
                         (head, tail) = os.path.split(filename)
                         command = "STOR " + tail
                         print('Uploading: ' + tail)
                         self.ftp.storbinary(command, self.file)
                         print('Upload Completed')
                     except ftplib.all_errors:
                         print('Error Uploading ' + tail)
                         self.status = 1
                         return self.status
                     self.file.close()
                     #back to initial directory in __init__()
                     self.ftp.cwd(self.remotefolder)
                     return self.status
                 def dir(self,remotefolder):
                     """
                     dir is used to change working directory of remote server and get folder and file list
                     Input:
                         remotefolder    - current working directory
                     Affects:
                         self.fileList    - file list of working directory
                     Return:
                         infoList    - list with filenames and size of file in bytes
                         self.folderList    -    folder list
                     """
                     self.remotefolder = remotefolder
                     print('Change to ' + self.remotefolder)
                     try:
                         self.ftp.cwd(remotefolder)
                     except ftplib.all_errors:
                         print('Error Change to ' + self.remotefolder)
                         infoList = None
                         self.folderList = None
                         return infoList,self.folderList
                     self.dirList = []
                     try:
                         self.dirList = self.ftp.nlst()
                     except ftplib.error_perm as resp:
                         if str(resp) == "550 No files found":
                                 print("no files in this directory")
                                 infoList = None
                                 self.folderList = None
                                 return infoList,self.folderList
                     except ftplib.all_errors:
                         print('Error Displaying Dir-Files')
                         infoList = None
                         self.folderList = None
                         return infoList,self.folderList
                     infoList = []
                     self.fileList = []
                     self.folderList = []
                     for f in self.dirList:
                         name,ext = os.path.splitext(f)
                         if ext != '':
                             self.fileList.append(f)
                             value = (f,self.ftp.size(f))
                             infoList.append(value)
                         if ext == '':
                             self.folderList.append(f)
                     return infoList,self.folderList
                 def close(self):
                     """
                     close is used to close and end FTP connection
                     Inputs: None
                     Return: void
                     """
                     self.ftp.close()
             class SendByFTP(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.status = 1
                     self.counter = 0
                 def error_print(self, ValueError):
                     print(ValueError, 'Error FTP')
                     print("don't worry the program is running...")
                 def worker_ftp(self, server, username, password, remotefolder, filenameList):
                     self.ftpClientObj = FTP(server, username, password, remotefolder)
                     for filename in filenameList:
                         self.ftpClientObj.upload(filename)
                     self.ftpClientObj.close()
                 def ftp_thread(self, server, username, password, remotefolder):
                     if not(self.status):
                         return
                     import multiprocessing
                     p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
                     p.start()
                     p.join(3)
                     if p.is_alive():
                         p.terminate()
                         p.join()
                         print('killing ftp process...')
                         self.status = 0
                         return
                     self.status = 1
                     return
                 def filterByExt(self, ext, localfolder):
                     fnameList = glob.glob1(localfolder,ext)
                     self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
                     if len(self.filenameList) == 0:
                         self.status = 0
                 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
                     self.counter += 1
                     if self.counter >= period:
                         self.filterByExt(ext, localfolder)
                         self.ftp_thread(server, username, password, remotefolder)
                         self.counter = 0
-                    self.status = 1
  No newline at end of file
+                    self.status = 1

schainpy/model/utils/jroutils_publish.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages