schain Commit - r1279:c53fe2a4a291 · Jicamarca Repository

Modificación a kmamisr para ejecutarse en la versión 3, creación de scripts con terminación v3 para difereciarlos, se comentó la linea #720 de JroIO_param.py debido a que reiniciaba la lista de archivos, ocasionando la reescritura del archivo hdf5. Alguna otra modificación aparente es producto de algunas variaciones en espacios al usar la función print()

joabAM -

r1279:c53fe2a4a291

parent child

Context file:

r1279:c53fe2a4a291

Collapse all files

schainpy/controller.py +5 0

             '''
             Updated on January , 2018, for multiprocessing purposes
             Author: Sergio Cortez
             Created on September , 2012
             '''
             from platform import python_version
             import sys
             import ast
             import datetime
             import traceback
             import math
             import time
             import zmq
             from multiprocessing import Process, Queue, Event, Value, cpu_count
             from threading import Thread
             from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
             from xml.dom import minidom
             from schainpy.admin import Alarm, SchainWarning
             from schainpy.model import *
             from schainpy.utils import log
             DTYPES = {
                 'Voltage': '.r',
                 'Spectra': '.pdata'
             }
             def MPProject(project, n=cpu_count()):
                 '''
                 Project wrapper to run schain in n processes
                 '''
                 rconf = project.getReadUnitObj()
                 op = rconf.getOperationObj('run')
                 dt1 = op.getParameterValue('startDate')
                 dt2 = op.getParameterValue('endDate')
                 tm1 = op.getParameterValue('startTime')
                 tm2 = op.getParameterValue('endTime')
                 days = (dt2 - dt1).days
                 for day in range(days + 1):
                     skip = 0
                     cursor = 0
                     processes = []
                     dt = dt1 + datetime.timedelta(day)
                     dt_str = dt.strftime('%Y/%m/%d')
                     reader = JRODataReader()
                     paths, files = reader.searchFilesOffLine(path=rconf.path,
                                                              startDate=dt,
                                                              endDate=dt,
                                                              startTime=tm1,
                                                              endTime=tm2,
                                                              ext=DTYPES[rconf.datatype])
                     nFiles = len(files)
                     if nFiles == 0:
                         continue
                     skip = int(math.ceil(nFiles / n))
                     while nFiles > cursor * skip:
                         rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
                                      skip=skip)
                         p = project.clone()
                         p.start()
                         processes.append(p)
                         cursor += 1
                     def beforeExit(exctype, value, trace):
                         for process in processes:
                             process.terminate()
                             process.join()
                         print(traceback.print_tb(trace))
                     sys.excepthook = beforeExit
                     for process in processes:
                         process.join()
                         process.terminate()
                     time.sleep(3)
             def wait(context):
                 time.sleep(1)
                 c = zmq.Context()
                 receiver = c.socket(zmq.SUB)
                 receiver.connect('ipc:///tmp/schain_{}_pub'.format(self.id))
                 receiver.setsockopt(zmq.SUBSCRIBE, self.id.encode())
                 msg = receiver.recv_multipart()[1]
                 context.terminate()
             class ParameterConf():
                 id = None
                 name = None
                 value = None
                 format = None
                 __formated_value = None
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.format = 'str'
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getValue(self):
                     value = self.value
                     format = self.format
                     if self.__formated_value != None:
                         return self.__formated_value
                     if format == 'obj':
                         return value
                     if format == 'str':
                         self.__formated_value = str(value)
                         return self.__formated_value
                     if value == '':
                         raise ValueError('%s: This parameter value is empty' % self.name)
                     if format == 'list':
                         strList = [s.strip() for s in value.split(',')]
                         self.__formated_value = strList
                         return self.__formated_value
                     if format == 'intlist':
                         '''
                         Example:
                             value = (0,1,2)
                         '''
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             new_value = [int(new_value)]
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'floatlist':
                         '''
                         Example:
                             value = (0.5, 1.4, 2.7)
                         '''
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             new_value = [float(new_value)]
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'date':
                         strList = value.split('/')
                         intList = [int(x) for x in strList]
                         date = datetime.date(intList[0], intList[1], intList[2])
                         self.__formated_value = date
                         return self.__formated_value
                     if format == 'time':
                         strList = value.split(':')
                         intList = [int(x) for x in strList]
                         time = datetime.time(intList[0], intList[1], intList[2])
                         self.__formated_value = time
                         return self.__formated_value
                     if format == 'pairslist':
                         '''
                         Example:
                             value = (0,1),(1,2)
                         '''
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             raise ValueError('%s has to be a tuple or list of pairs' % value)
                         if type(new_value[0]) not in (tuple, list):
                             if len(new_value) != 2:
                                 raise ValueError('%s has to be a tuple or list of pairs' % value)
                             new_value = [new_value]
                         for thisPair in new_value:
                             if len(thisPair) != 2:
                                 raise ValueError('%s has to be a tuple or list of pairs' % value)
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'multilist':
                         '''
                         Example:
                             value = (0,1,2),(3,4,5)
                         '''
                         multiList = ast.literal_eval(value)
                         if type(multiList[0]) == int:
                             multiList = ast.literal_eval('(' + value + ')')
                         self.__formated_value = multiList
                         return self.__formated_value
                     if format == 'bool':
                         value = int(value)
                     if format == 'int':
                         value = float(value)
                     format_func = eval(format)
                     self.__formated_value = format_func(value)
                     return self.__formated_value
                 def updateId(self, new_id):
                     self.id = str(new_id)
                 def setup(self, id, name, value, format='str'):
                     self.id = str(id)
                     self.name = name
                     if format == 'obj':
                         self.value = value
                     else:
                         self.value = str(value)
                     self.format = str.lower(format)
                     self.getValue()
                     return 1
                 def update(self, name, value, format='str'):
                     self.name = name
                     self.value = str(value)
                     self.format = format
                 def makeXml(self, opElement):
                     if self.name not in ('queue',):
                         parmElement = SubElement(opElement, self.ELEMENTNAME)
                         parmElement.set('id', str(self.id))
                         parmElement.set('name', self.name)
                         parmElement.set('value', self.value)
                         parmElement.set('format', self.format)
                 def readXml(self, parmElement):
                     self.id = parmElement.get('id')
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                     self.format = str.lower(parmElement.get('format'))
                     # Compatible with old signal chain version
                     if self.format == 'int' and self.name == 'idfigure':
                         self.name = 'id'
                 def printattr(self):
                     print('Parameter[%s]: name = %s, value = %s, format = %s, project_id = %s' % (self.id, self.name, self.value, self.format, self.project_id))
             class OperationConf():
                 ELEMENTNAME = 'Operation'
                 def __init__(self):
                     self.id = '0'
                     self.name = None
                     self.priority = None
                     self.topic = None
                 def __getNewId(self):
                     return int(self.id) * 10 + len(self.parmConfObjList) + 1
                 def getId(self):
                     return self.id
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     n = 1
                     for parmObj in self.parmConfObjList:
                         idParm = str(int(new_id) * 10 + n)
                         parmObj.updateId(idParm)
                         n += 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getParameterObjList(self):
                     return self.parmConfObjList
                 def getParameterObj(self, parameterName):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.name != parameterName:
                             continue
                         return parmConfObj
                     return None
                 def getParameterObjfromValue(self, parameterValue):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.getValue() != parameterValue:
                             continue
                         return parmConfObj.getValue()
                     return None
                 def getParameterValue(self, parameterName):
                     parameterObj = self.getParameterObj(parameterName)
                 #         if not parameterObj:
                 #             return None
                     value = parameterObj.getValue()
                     return value
                 def getKwargs(self):
                     kwargs = {}
                     for parmConfObj in self.parmConfObjList:
                         if self.name == 'run' and parmConfObj.name == 'datatype':
                             continue
                         kwargs[parmConfObj.name] = parmConfObj.getValue()
                     return kwargs
                 def setup(self, id, name, priority, type, project_id, err_queue, lock):
                     self.id = str(id)
                     self.project_id = project_id
                     self.name = name
                     self.type = type
                     self.priority = priority
                     self.err_queue = err_queue
                     self.lock = lock
                     self.parmConfObjList = []
                 def removeParameters(self):
                     for obj in self.parmConfObjList:
                         del obj
                     self.parmConfObjList = []
                 def addParameter(self, name, value, format='str'):
                     if value is None:
                         return None
                     id = self.__getNewId()
                     parmConfObj = ParameterConf()
                     if not parmConfObj.setup(id, name, value, format):
                         return None
                     self.parmConfObjList.append(parmConfObj)
                     return parmConfObj
                 def changeParameter(self, name, value, format='str'):
                     parmConfObj = self.getParameterObj(name)
                     parmConfObj.update(name, value, format)
                     return parmConfObj
                 def makeXml(self, procUnitElement):
                     opElement = SubElement(procUnitElement, self.ELEMENTNAME)
                     opElement.set('id', str(self.id))
                     opElement.set('name', self.name)
                     opElement.set('type', self.type)
                     opElement.set('priority', str(self.priority))
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.makeXml(opElement)
                 def readXml(self, opElement, project_id):
                     self.id = opElement.get('id')
                     self.name = opElement.get('name')
                     self.type = opElement.get('type')
                     self.priority = opElement.get('priority')
                     self.project_id = str(project_id)
                     # Compatible with old signal chain version
                     # Use of 'run' method instead 'init'
                     if self.type == 'self' and self.name == 'init':
                         self.name = 'run'
                     self.parmConfObjList = []
                     parmElementList = opElement.iter(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         # Compatible with old signal chain version
                         # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
                         if self.type != 'self' and self.name == 'Plot':
                             if parmConfObj.format == 'str' and parmConfObj.name == 'type':
                                 self.name = parmConfObj.value
                                 continue
                         self.parmConfObjList.append(parmConfObj)
                 def printattr(self):
                     print('%s[%s]: name = %s, type = %s, priority = %s, project_id = %s' % (self.ELEMENTNAME,
                                                                            self.id,
                                                                            self.name,
                                                                            self.type,
                                                                            self.priority,
                                                                            self.project_id))
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.printattr()
                 def createObject(self):
                     className = eval(self.name)
                     if self.type == 'other':
                         opObj = className()
                     elif self.type == 'external':
                         kwargs = self.getKwargs()
                         opObj = className(self.id, self.id, self.project_id, self.err_queue, self.lock, 'Operation', **kwargs)
                         opObj.start()
                         self.opObj = opObj
                     return opObj
             class ProcUnitConf():
                 ELEMENTNAME = 'ProcUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.opConfObjList = []
                     self.procUnitObj = None
                     self.opObjDict = {}
                 def __getPriority(self):
                     return len(self.opConfObjList) + 1
                 def __getNewId(self):
                     return int(self.id) * 10 + len(self.opConfObjList) + 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def updateId(self, new_id):
                     '''
                     new_id = int(parentId) * 10 + (int(self.id) % 10)
                     new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
                     # If this proc unit has not inputs
                     #if self.inputId == '0':
                         #new_inputId = 0
                     n = 1
                     for opConfObj in self.opConfObjList:
                         idOp = str(int(new_id) * 10 + n)
                         opConfObj.updateId(idOp)
                         n += 1
                     self.parentId = str(parentId)
                     self.id = str(new_id)
                     #self.inputId = str(new_inputId)
                     '''
                     n = 1
                 def getInputId(self):
                     return self.inputId
                 def getOperationObjList(self):
                     return self.opConfObjList
                 def getOperationObj(self, name=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.name != name:
                             continue
                         return opConfObj
                     return None
                 def getOpObjfromParamValue(self, value=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
                             continue
                         return opConfObj
                     return None
                 def getProcUnitObj(self):
                     return self.procUnitObj
                 def setup(self, project_id, id, name, datatype, inputId, err_queue, lock):
                     '''
                         id sera el topico a publicar
                         inputId sera el topico a subscribirse
                     '''
                     # Compatible with old signal chain version
                     if datatype == None and name == None:
                         raise ValueError('datatype or name should be defined')
                     #Definir una condicion para inputId cuando sea 0
                     if name == None:
                         if 'Proc' in datatype:
                             name = datatype
                         else:
                             name = '%sProc' % (datatype)
                     if datatype == None:
                         datatype = name.replace('Proc', '')
                     self.id = str(id)
                     self.project_id = project_id
                     self.name = name
                     self.datatype = datatype
                     self.inputId = inputId
                     self.err_queue = err_queue
                     self.lock = lock
                     self.opConfObjList = []
                     self.addOperation(name='run', optype='self')
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                     self.addOperation(name='run')
                 def addParameter(self, **kwargs):
                     '''
                     Add parameters to 'run' operation
                     '''
                     opObj = self.opConfObjList[0]
                     opObj.addParameter(**kwargs)
                     return opObj
                 def addOperation(self, name, optype='self'):
                     '''
                         Actualizacion - > proceso comunicacion
                         En el caso de optype='self', elminar. DEfinir comuncacion IPC -> Topic
                         definir el tipoc de socket o comunicacion ipc++
                     '''
                     id = self.__getNewId()
                     priority = self.__getPriority() # Sin mucho sentido, pero puede usarse
                     opConfObj = OperationConf()
                     opConfObj.setup(id, name=name, priority=priority, type=optype, project_id=self.project_id, err_queue=self.err_queue, lock=self.lock)
                     self.opConfObjList.append(opConfObj)
                     return opConfObj
                 def makeXml(self, projectElement):
                     procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
                     procUnitElement.set('id', str(self.id))
                     procUnitElement.set('name', self.name)
                     procUnitElement.set('datatype', self.datatype)
                     procUnitElement.set('inputId', str(self.inputId))
                     for opConfObj in self.opConfObjList:
                         opConfObj.makeXml(procUnitElement)
                 def readXml(self, upElement, project_id):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.inputId = upElement.get('inputId')
                     self.project_id = str(project_id)
                     if self.ELEMENTNAME == 'ReadUnit':
                         self.datatype = self.datatype.replace('Reader', '')
                     if self.ELEMENTNAME == 'ProcUnit':
                         self.datatype = self.datatype.replace('Proc', '')
                     if self.inputId == 'None':
                         self.inputId = '0'
                     self.opConfObjList = []
                     opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement, project_id)
                         self.opConfObjList.append(opConfObj)
                 def printattr(self):
                     print('%s[%s]: name = %s, datatype = %s, inputId = %s, project_id = %s' % (self.ELEMENTNAME,
                                                                               self.id,
                                                                               self.name,
                                                                               self.datatype,
                                                                               self.inputId,
                                                                               self.project_id))
                     for opConfObj in self.opConfObjList:
                         opConfObj.printattr()
                 def getKwargs(self):
                     opObj = self.opConfObjList[0]
                     kwargs = opObj.getKwargs()
                     return kwargs
                 def createObjects(self):
                     '''
                     Instancia de unidades de procesamiento.
                     '''
                     className = eval(self.name)
+                    #print(self.name)
                     kwargs = self.getKwargs()
+                    #print(kwargs)
+                    #print("mark_a")
                     procUnitObj = className(self.id, self.inputId, self.project_id, self.err_queue, self.lock, 'ProcUnit', **kwargs)
+                    #print("mark_b")
                     log.success('creating process...', self.name)
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self' and opConfObj.name == 'run':
                             continue
                         elif opConfObj.type == 'self':
                             opObj = getattr(procUnitObj, opConfObj.name)
                         else:
                             opObj = opConfObj.createObject()
                         log.success('adding operation: {}, type:{}'.format(
                             opConfObj.name,
                             opConfObj.type), self.name)
                         procUnitObj.addOperation(opConfObj, opObj)
                     procUnitObj.start()
                     self.procUnitObj = procUnitObj
                 def close(self):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self':
                             continue
                         opObj = self.procUnitObj.getOperationObj(opConfObj.id)
                         opObj.close()
                     self.procUnitObj.close()
                     return
             class ReadUnitConf(ProcUnitConf):
                 ELEMENTNAME = 'ReadUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.opConfObjList = []
                     self.lock = Event()
                     self.lock.set()
                     self.lock.n = Value('d', 0)
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
                           startTime='', endTime='', server=None, **kwargs):
                     '''
                     *****el id del proceso sera el Topico
                     Adicion de {topic}, si no esta presente -> error
                     kwargs deben ser trasmitidos en la instanciacion
                     '''
                     # Compatible with old signal chain version
                     if datatype == None and name == None:
                         raise ValueError('datatype or name should be defined')
                     if name == None:
                         if 'Reader' in datatype:
                             name = datatype
                             datatype = name.replace('Reader','')
                         else:
                             name = '{}Reader'.format(datatype)
                     if datatype == None:
                         if 'Reader' in name:
                             datatype = name.replace('Reader','')
                         else:
                             datatype = name
                             name = '{}Reader'.format(name)
                     self.id = id
                     self.project_id = project_id
                     self.name = name
                     self.datatype = datatype
                     if path != '':
                         self.path = os.path.abspath(path)
+                    print (self.path)
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.server = server
                     self.err_queue = err_queue
                     self.addRunOperation(**kwargs)
                 def update(self, **kwargs):
                     if 'datatype' in kwargs:
                         datatype = kwargs.pop('datatype')
                         if 'Reader' in datatype:
                             self.name = datatype
                         else:
                             self.name = '%sReader' % (datatype)
                         self.datatype = self.name.replace('Reader', '')
                     attrs = ('path', 'startDate', 'endDate',
                              'startTime', 'endTime')
                     for attr in attrs:
                         if attr in kwargs:
                             setattr(self, attr, kwargs.pop(attr))
                     self.updateRunOperation(**kwargs)
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                 def addRunOperation(self, **kwargs):
                     opObj = self.addOperation(name='run', optype='self')
                     if self.server is None:
                         opObj.addParameter(
                             name='datatype', value=self.datatype, format='str')
                         opObj.addParameter(name='path', value=self.path, format='str')
                         opObj.addParameter(
                             name='startDate', value=self.startDate, format='date')
                         opObj.addParameter(
                             name='endDate', value=self.endDate, format='date')
                         opObj.addParameter(
                             name='startTime', value=self.startTime, format='time')
                         opObj.addParameter(
                             name='endTime', value=self.endTime, format='time')
                         for key, value in list(kwargs.items()):
                             opObj.addParameter(name=key, value=value,
                                                format=type(value).__name__)
                     else:
                         opObj.addParameter(name='server', value=self.server, format='str')
                     return opObj
                 def updateRunOperation(self, **kwargs):
                     opObj = self.getOperationObj(name='run')
                     opObj.removeParameters()
                     opObj.addParameter(name='datatype', value=self.datatype, format='str')
                     opObj.addParameter(name='path', value=self.path, format='str')
                     opObj.addParameter(
                         name='startDate', value=self.startDate, format='date')
                     opObj.addParameter(name='endDate', value=self.endDate, format='date')
                     opObj.addParameter(
                         name='startTime', value=self.startTime, format='time')
                     opObj.addParameter(name='endTime', value=self.endTime, format='time')
                     for key, value in list(kwargs.items()):
                         opObj.addParameter(name=key, value=value,
                                            format=type(value).__name__)
                     return opObj
                 def readXml(self, upElement, project_id):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.project_id = str(project_id)  #yong
                     if self.ELEMENTNAME == 'ReadUnit':
                         self.datatype = self.datatype.replace('Reader', '')
                     self.opConfObjList = []
                     opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement, project_id)
                         self.opConfObjList.append(opConfObj)
                         if opConfObj.name == 'run':
                             self.path = opConfObj.getParameterValue('path')
                             self.startDate = opConfObj.getParameterValue('startDate')
                             self.endDate = opConfObj.getParameterValue('endDate')
                             self.startTime = opConfObj.getParameterValue('startTime')
                             self.endTime = opConfObj.getParameterValue('endTime')
             class Project(Process):
                 ELEMENTNAME = 'Project'
                 def __init__(self):
                     Process.__init__(self)
                     self.id = None
                     self.filename = None
                     self.description = None
                     self.email = None
                     self.alarm = None
                     self.procUnitConfObjDict = {}
                     self.err_queue = Queue()
                 def __getNewId(self):
                     idList = list(self.procUnitConfObjDict.keys())
                     id = int(self.id) * 10
                     while True:
                         id += 1
                         if str(id) in idList:
                             continue
                         break
                     return str(id)
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     keyList = list(self.procUnitConfObjDict.keys())
                     keyList.sort()
                     n = 1
                     newProcUnitConfObjDict = {}
                     for procKey in keyList:
                         procUnitConfObj = self.procUnitConfObjDict[procKey]
                         idProcUnit = str(int(self.id) * 10 + n)
                         procUnitConfObj.updateId(idProcUnit)
                         newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
                         n += 1
                     self.procUnitConfObjDict = newProcUnitConfObjDict
                 def setup(self, id=1, name='', description='', email=None, alarm=[]):
                     print(' ')
                     print('*' * 60)
                     print('*  Starting SIGNAL CHAIN PROCESSING (Multiprocessing) v%s *' % schainpy.__version__)
                     print('*' * 60)
                     print("*  Python " + python_version() + "  *")
                     print('*' * 19)
                     print(' ')
                     self.id = str(id)
                     self.description = description
                     self.email = email
                     self.alarm = alarm
                     if name:
                         self.name = '{} ({})'.format(Process.__name__, name)
                 def update(self, **kwargs):
                     for key, value in list(kwargs.items()):
                         setattr(self, key, value)
                 def clone(self):
                     p = Project()
                     p.procUnitConfObjDict = self.procUnitConfObjDict
                     return p
                 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
                     '''
                     Actualizacion:
                         Se agrego un nuevo argumento: topic -relativo a la forma de comunicar los procesos simultaneos
                         * El id del proceso sera el topico al que se deben subscribir los procUnits para recibir la informacion(data)
                     '''
                     if id is None:
                         idReadUnit = self.__getNewId()
                     else:
                         idReadUnit = str(id)
                     readUnitConfObj = ReadUnitConf()
                     readUnitConfObj.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
                     self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     return readUnitConfObj
                 def addProcUnit(self, inputId='0', datatype=None, name=None):
                     '''
                     Actualizacion:
                         Se agrego dos nuevos argumentos: topic_read (lee data de otro procUnit) y topic_write(escribe o envia data a otro procUnit)
                         Deberia reemplazar a "inputId"
                         ** A fin de mantener el inputID, este sera la representaacion del topicoal que deben subscribirse. El ID propio de la intancia
                         (proceso) sera el topico de la publicacion, todo sera asignado de manera dinamica.
                     '''
                     idProcUnit = self.__getNewId()
                     procUnitConfObj = ProcUnitConf()
                     input_proc = self.procUnitConfObjDict[inputId]
                     procUnitConfObj.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue, input_proc.lock)
                     self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     return procUnitConfObj
                 def removeProcUnit(self, id):
                     if id in list(self.procUnitConfObjDict.keys()):
                         self.procUnitConfObjDict.pop(id)
                 def getReadUnitId(self):
                     readUnitConfObj = self.getReadUnitObj()
                     return readUnitConfObj.id
                 def getReadUnitObj(self):
                     for obj in list(self.procUnitConfObjDict.values()):
                         if obj.getElementName() == 'ReadUnit':
                             return obj
                     return None
                 def getProcUnitObj(self, id=None, name=None):
                     if id != None:
                         return self.procUnitConfObjDict[id]
                     if name != None:
                         return self.getProcUnitObjByName(name)
                     return None
                 def getProcUnitObjByName(self, name):
                     for obj in list(self.procUnitConfObjDict.values()):
                         if obj.name == name:
                             return obj
                     return None
                 def procUnitItems(self):
                     return list(self.procUnitConfObjDict.items())
                 def makeXml(self):
                     projectElement = Element('Project')
                     projectElement.set('id', str(self.id))
                     projectElement.set('name', self.name)
                     projectElement.set('description', self.description)
                     for procUnitConfObj in list(self.procUnitConfObjDict.values()):
                         procUnitConfObj.makeXml(projectElement)
                     self.projectElement = projectElement
                 def writeXml(self, filename=None):
                     if filename == None:
                         if self.filename:
                             filename = self.filename
                         else:
                             filename = 'schain.xml'
                     if not filename:
                         print('filename has not been defined. Use setFilename(filename) for do it.')
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.access(os.path.dirname(abs_file), os.W_OK):
                         print('No write permission on %s' % os.path.dirname(abs_file))
                         return 0
                     if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
                         print('File %s already exists and it could not be overwriten' % abs_file)
                         return 0
                     self.makeXml()
                     ElementTree(self.projectElement).write(abs_file, method='xml')
                     self.filename = abs_file
                     return 1
                 def readXml(self, filename=None):
                     if not filename:
                         print('filename is not defined')
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.path.isfile(abs_file):
                         print('%s file does not exist' % abs_file)
                         return 0
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     try:
                         self.projectElement = ElementTree().parse(abs_file)
                     except:
                         print('Error reading %s, verify file format' % filename)
                         return 0
                     self.project = self.projectElement.tag
                     self.id = self.projectElement.get('id')
                     self.name = self.projectElement.get('name')
                     self.description = self.projectElement.get('description')
                     readUnitElementList = self.projectElement.iter(
                         ReadUnitConf().getElementName())
                     for readUnitElement in readUnitElementList:
                         readUnitConfObj = ReadUnitConf()
                         readUnitConfObj.readXml(readUnitElement, self.id)
                         self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     procUnitElementList = self.projectElement.iter(
                         ProcUnitConf().getElementName())
                     for procUnitElement in procUnitElementList:
                         procUnitConfObj = ProcUnitConf()
                         procUnitConfObj.readXml(procUnitElement, self.id)
                         self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     self.filename = abs_file
                     return 1
                 def __str__(self):
                     print('Project: name = %s, description = %s, id = %s' % (
                                                                         self.name,
                                                                         self.description,
                                                                         self.id))
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         print(procUnitConfObj)
                 def createObjects(self):
                     keys = list(self.procUnitConfObjDict.keys())
                     keys.sort()
                     for key in keys:
                         self.procUnitConfObjDict[key].createObjects()
                 def monitor(self):
                     t = Thread(target=self.__monitor, args=(self.err_queue, self.ctx))
                     t.start()
                 def __monitor(self, queue, ctx):
                     import socket
                     procs = 0
                     err_msg = ''
                     while True:
                         msg = queue.get()
                         if '#_start_#' in msg:
                             procs += 1
                         elif '#_end_#' in msg:
                             procs -=1
                         else:
                             err_msg = msg
                         if procs == 0 or 'Traceback' in err_msg:
                             break
                         time.sleep(0.1)
                     if '|' in err_msg:
                         name, err = err_msg.split('|')
                         if 'SchainWarning' in err:
                             log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
                         elif 'SchainError' in err:
                             log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
                         else:
                             log.error(err, name)
                     else:
                         name, err = self.name, err_msg
                     time.sleep(2)
                     for conf in self.procUnitConfObjDict.values():
                         for confop in conf.opConfObjList:
                             if confop.type == 'external':
                                 confop.opObj.terminate()
                         conf.procUnitObj.terminate()
                     ctx.term()
                     message = ''.join(err)
                     if err_msg:
                         subject = 'SChain v%s: Error running %s\n' % (
                             schainpy.__version__, self.name)
                         subtitle = 'Hostname: %s\n' % socket.gethostbyname(
                             socket.gethostname())
                         subtitle += 'Working directory: %s\n' % os.path.abspath('./')
                         subtitle += 'Configuration file: %s\n' % self.filename
                         subtitle += 'Time: %s\n' % str(datetime.datetime.now())
                         readUnitConfObj = self.getReadUnitObj()
                         if readUnitConfObj:
                             subtitle += '\nInput parameters:\n'
                             subtitle += '[Data path = %s]\n' % readUnitConfObj.path
                             subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
                             subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
                             subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
                             subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
                             subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
                         a = Alarm(
                             modes=self.alarm,
                             email=self.email,
                             message=message,
                             subject=subject,
                             subtitle=subtitle,
                             filename=self.filename
                         )
                         a.start()
                 def isPaused(self):
                     return 0
                 def isStopped(self):
                     return 0
                 def runController(self):
                     '''
                     returns 0 when this process has been stopped, 1 otherwise
                     '''
                     if self.isPaused():
                         print('Process suspended')
                         while True:
                             time.sleep(0.1)
                             if not self.isPaused():
                                 break
                             if self.isStopped():
                                 break
                         print('Process reinitialized')
                     if self.isStopped():
                         print('Process stopped')
                         return 0
                     return 1
                 def setFilename(self, filename):
                     self.filename = filename
                 def setProxy(self):
                     if not os.path.exists('/tmp/schain'):
                         os.mkdir('/tmp/schain')
                     self.ctx = zmq.Context()
                     xpub = self.ctx.socket(zmq.XPUB)
                     xpub.bind('ipc:///tmp/schain/{}_pub'.format(self.id))
                     xsub = self.ctx.socket(zmq.XSUB)
                     xsub.bind('ipc:///tmp/schain/{}_sub'.format(self.id))
                     self.monitor()
                     try:
                         zmq.proxy(xpub, xsub)
                     except zmq.ContextTerminated:
                         xpub.close()
                         xsub.close()
                 def run(self):
                     log.success('Starting {}: {}'.format(self.name, self.id), tag='')
                     self.start_time = time.time()
                     self.createObjects()
                     self.setProxy()
                     log.success('{} Done (Time: {}s)'.format(
                         self.name,
                         time.time()-self.start_time), '')

schainpy/model/data/jrodata.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file

NO CONTENT: modified file

schainpy/model/data/jroheaderIO.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file

NO CONTENT: modified file

schainpy/model/graphics/jroplot_base.py +2 -4

             import os
             import sys
             import zmq
             import time
             import numpy
             import datetime
             from functools import wraps
             from threading import Thread
             import matplotlib
             if 'BACKEND' in os.environ:
                 matplotlib.use(os.environ['BACKEND'])
             elif 'linux' in sys.platform:
                 matplotlib.use("TkAgg")
             elif 'darwin' in sys.platform:
                 matplotlib.use('WxAgg')
             else:
                 from schainpy.utils import log
                 log.warning('Using default Backend="Agg"', 'INFO')
                 matplotlib.use('Agg')
             import matplotlib.pyplot as plt
             from matplotlib.patches import Polygon
             from mpl_toolkits.axes_grid1 import make_axes_locatable
             from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
             from schainpy.model.data.jrodata import PlotterData
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.utils import log
             jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
             blu_values = matplotlib.pyplot.get_cmap(
                 'seismic_r', 20)(numpy.arange(20))[10:15]
             ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
                 'jro', numpy.vstack((blu_values, jet_values)))
             matplotlib.pyplot.register_cmap(cmap=ncmap)
             CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis',
                                                'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
             EARTH_RADIUS = 6.3710e3
             def ll2xy(lat1, lon1, lat2, lon2):
                 p = 0.017453292519943295
                 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
                     numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
                 r = 12742 * numpy.arcsin(numpy.sqrt(a))
                 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
                                       * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
                 theta = -theta + numpy.pi/2
                 return r*numpy.cos(theta), r*numpy.sin(theta)
             def km2deg(km):
                 '''
                 Convert distance in km to degrees
                 '''
                 return numpy.rad2deg(km/EARTH_RADIUS)
             def figpause(interval):
                 backend = plt.rcParams['backend']
                 if backend in matplotlib.rcsetup.interactive_bk:
                     figManager = matplotlib._pylab_helpers.Gcf.get_active()
                     if figManager is not None:
                         canvas = figManager.canvas
                         if canvas.figure.stale:
                             canvas.draw()
                         try:
                             canvas.start_event_loop(interval)
                         except:
                             pass
                         return
             def popup(message):
                 '''
                 '''
                 fig = plt.figure(figsize=(12, 8), facecolor='r')
                 text = '\n'.join([s.strip() for s in message.split(':')])
                 fig.text(0.01, 0.5, text, ha='left', va='center',
                          size='20', weight='heavy', color='w')
                 fig.show()
                 figpause(1000)
             class Throttle(object):
                 '''
                 Decorator that prevents a function from being called more than once every
                 time period.
                 To create a function that cannot be called more than once a minute, but
                 will sleep until it can be called:
                 @Throttle(minutes=1)
                 def foo():
                   pass
                 for i in range(10):
                   foo()
                   print "This function has run %s times." % i
                 '''
                 def __init__(self, seconds=0, minutes=0, hours=0):
                     self.throttle_period = datetime.timedelta(
                         seconds=seconds, minutes=minutes, hours=hours
                     )
                     self.time_of_last_call = datetime.datetime.min
                 def __call__(self, fn):
                     @wraps(fn)
                     def wrapper(*args, **kwargs):
                         coerce = kwargs.pop('coerce', None)
                         if coerce:
                             self.time_of_last_call = datetime.datetime.now()
                             return fn(*args, **kwargs)
                         else:
                             now = datetime.datetime.now()
                             time_since_last_call = now - self.time_of_last_call
                             time_left = self.throttle_period - time_since_last_call
                             if time_left > datetime.timedelta(seconds=0):
                                 return
                         self.time_of_last_call = datetime.datetime.now()
                         return fn(*args, **kwargs)
                     return wrapper
             def apply_throttle(value):
                 @Throttle(seconds=value)
                 def fnThrottled(fn):
                     fn()
                 return fnThrottled
             @MPDecorator
             class Plot(Operation):
                 '''
                 Base class for Schain plotting operations
                 '''
                 CODE = 'Figure'
                 colormap = 'jet'
                 bgcolor = 'white'
                 __missing = 1E30
                 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
                              'zlimits', 'xlabel', 'ylabel', 'xaxis', 'cb_label', 'title',
                              'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',
                              'showprofile', 'decimation', 'pause']
                 def __init__(self):
                     Operation.__init__(self)
                     self.isConfig = False
                     self.isPlotConfig = False
                     self.save_counter = 1
                     self.sender_counter = 1
                     self.data = None
                 def __fmtTime(self, x, pos):
                     '''
                     '''
                     return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
                 def __setup(self, **kwargs):
                     '''
                     Initialize variables
                     '''
                     self.figures = []
                     self.axes = []
                     self.cb_axes = []
                     self.localtime = kwargs.pop('localtime', True)
                     self.show = kwargs.get('show', True)
                     self.save = kwargs.get('save', False)
                     self.save_period = kwargs.get('save_period', 1)
                     self.ftp = kwargs.get('ftp', False)
                     self.colormap = kwargs.get('colormap', self.colormap)
                     self.colormap_coh = kwargs.get('colormap_coh', 'jet')
                     self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
                     self.colormaps = kwargs.get('colormaps', None)
                     self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
                     self.showprofile = kwargs.get('showprofile', False)
                     self.title = kwargs.get('wintitle', self.CODE.upper())
                     self.cb_label = kwargs.get('cb_label', None)
                     self.cb_labels = kwargs.get('cb_labels', None)
                     self.labels = kwargs.get('labels', None)
                     self.xaxis = kwargs.get('xaxis', 'frequency')
                     self.zmin = kwargs.get('zmin', None)
                     self.zmax = kwargs.get('zmax', None)
                     self.zlimits = kwargs.get('zlimits', None)
                     self.xmin = kwargs.get('xmin', None)
                     self.xmax = kwargs.get('xmax', None)
                     self.xrange = kwargs.get('xrange', 24)
                     self.xscale = kwargs.get('xscale', None)
                     self.ymin = kwargs.get('ymin', None)
                     self.ymax = kwargs.get('ymax', None)
                     self.yscale = kwargs.get('yscale', None)
                     self.xlabel = kwargs.get('xlabel', None)
                     self.decimation = kwargs.get('decimation', None)
                     self.showSNR = kwargs.get('showSNR', False)
                     self.oneFigure = kwargs.get('oneFigure', True)
                     self.width = kwargs.get('width', None)
                     self.height = kwargs.get('height', None)
                     self.colorbar = kwargs.get('colorbar', True)
                     self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
                     self.channels = kwargs.get('channels', None)
                     self.titles = kwargs.get('titles', [])
                     self.polar = False
                     self.type = kwargs.get('type', 'iq')
                     self.grid = kwargs.get('grid', False)
                     self.pause = kwargs.get('pause', False)
                     self.save_labels = kwargs.get('save_labels', None)
                     self.realtime = kwargs.get('realtime', True)
                     self.buffering = kwargs.get('buffering', True)
                     self.throttle = kwargs.get('throttle', 2)
                     self.exp_code = kwargs.get('exp_code', None)
                     self.plot_server = kwargs.get('plot_server', False)
                     self.sender_period = kwargs.get('sender_period', 1)
                     self.__throttle_plot = apply_throttle(self.throttle)
                     self.data = PlotterData(
                         self.CODE, self.throttle, self.exp_code, self.buffering, snr=self.showSNR)
                     if self.plot_server:
                         if not self.plot_server.startswith('tcp://'):
                             self.plot_server = 'tcp://{}'.format(self.plot_server)
                         log.success(
                             'Sending to server: {}'.format(self.plot_server),
                             self.name
                         )
                     if 'plot_name' in kwargs:
                         self.plot_name = kwargs['plot_name']
                 def __setup_plot(self):
                     '''
                     Common setup for all figures, here figures and axes are created
                     '''
                     self.setup()
                     self.time_label = 'LT' if self.localtime else 'UTC'
                     if self.width is None:
                         self.width = 8
                     self.figures = []
                     self.axes = []
                     self.cb_axes = []
                     self.pf_axes = []
                     self.cmaps = []
                     size = '15%' if self.ncols == 1 else '30%'
                     pad = '4%' if self.ncols == 1 else '8%'
                     if self.oneFigure:
                         if self.height is None:
                             self.height = 1.4 * self.nrows + 1
                         fig = plt.figure(figsize=(self.width, self.height),
                                          edgecolor='k',
                                          facecolor='w')
                         self.figures.append(fig)
                         for n in range(self.nplots):
                             ax = fig.add_subplot(self.nrows, self.ncols,
                                                  n + 1, polar=self.polar)
                             ax.tick_params(labelsize=8)
                             ax.firsttime = True
                             ax.index = 0
                             ax.press = None
                             self.axes.append(ax)
                             if self.showprofile:
                                 cax = self.__add_axes(ax, size=size, pad=pad)
                                 cax.tick_params(labelsize=8)
                                 self.pf_axes.append(cax)
                     else:
                         if self.height is None:
                             self.height = 3
                         for n in range(self.nplots):
                             fig = plt.figure(figsize=(self.width, self.height),
                                              edgecolor='k',
                                              facecolor='w')
                             ax = fig.add_subplot(1, 1, 1, polar=self.polar)
                             ax.tick_params(labelsize=8)
                             ax.firsttime = True
                             ax.index = 0
                             ax.press = None
                             self.figures.append(fig)
                             self.axes.append(ax)
                             if self.showprofile:
                                 cax = self.__add_axes(ax, size=size, pad=pad)
                                 cax.tick_params(labelsize=8)
                                 self.pf_axes.append(cax)
                     for n in range(self.nrows):
                         if self.colormaps is not None:
                             cmap = plt.get_cmap(self.colormaps[n])
                         else:
                             cmap = plt.get_cmap(self.colormap)
                         cmap.set_bad(self.bgcolor, 1.)
                         self.cmaps.append(cmap)
                     for fig in self.figures:
                         fig.canvas.mpl_connect('key_press_event', self.OnKeyPress)
                         fig.canvas.mpl_connect('scroll_event', self.OnBtnScroll)
                         fig.canvas.mpl_connect('button_press_event', self.onBtnPress)
                         fig.canvas.mpl_connect('motion_notify_event', self.onMotion)
                         fig.canvas.mpl_connect('button_release_event', self.onBtnRelease)
                 def OnKeyPress(self, event):
                     '''
                     Event for pressing keys (up, down) change colormap
                     '''
                     ax = event.inaxes
                     if ax in self.axes:
                         if event.key == 'down':
                             ax.index += 1
                         elif event.key == 'up':
                             ax.index -= 1
                         if ax.index < 0:
                             ax.index = len(CMAPS) - 1
                         elif ax.index == len(CMAPS):
                             ax.index = 0
                         cmap = CMAPS[ax.index]
                         ax.cbar.set_cmap(cmap)
                         ax.cbar.draw_all()
                         ax.plt.set_cmap(cmap)
                         ax.cbar.patch.figure.canvas.draw()
                         self.colormap = cmap.name
                 def OnBtnScroll(self, event):
                     '''
                     Event for scrolling, scale figure
                     '''
                     cb_ax = event.inaxes
                     if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
                         ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
                         pt = ax.cbar.ax.bbox.get_points()[:, 1]
                         nrm = ax.cbar.norm
                         vmin, vmax, p0, p1, pS = (
                             nrm.vmin, nrm.vmax, pt[0], pt[1], event.y)
                         scale = 2 if event.step == 1 else 0.5
                         point = vmin + (vmax - vmin) / (p1 - p0) * (pS - p0)
                         ax.cbar.norm.vmin = point - scale * (point - vmin)
                         ax.cbar.norm.vmax = point - scale * (point - vmax)
                         ax.plt.set_norm(ax.cbar.norm)
                         ax.cbar.draw_all()
                         ax.cbar.patch.figure.canvas.draw()
                 def onBtnPress(self, event):
                     '''
                     Event for mouse button press
                     '''
                     cb_ax = event.inaxes
                     if cb_ax is None:
                         return
                     if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
                         cb_ax.press = event.x, event.y
                     else:
                         cb_ax.press = None
                 def onMotion(self, event):
                     '''
                     Event for move inside colorbar
                     '''
                     cb_ax = event.inaxes
                     if cb_ax is None:
                         return
                     if cb_ax not in [ax.cbar.ax for ax in self.axes if ax.cbar]:
                         return
                     if cb_ax.press is None:
                         return
                     ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
                     xprev, yprev = cb_ax.press
                     dx = event.x - xprev
                     dy = event.y - yprev
                     cb_ax.press = event.x, event.y
                     scale = ax.cbar.norm.vmax - ax.cbar.norm.vmin
                     perc = 0.03
                     if event.button == 1:
                         ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
                         ax.cbar.norm.vmax -= (perc * scale) * numpy.sign(dy)
                     elif event.button == 3:
                         ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
                         ax.cbar.norm.vmax += (perc * scale) * numpy.sign(dy)
                     ax.cbar.draw_all()
                     ax.plt.set_norm(ax.cbar.norm)
                     ax.cbar.patch.figure.canvas.draw()
                 def onBtnRelease(self, event):
                     '''
                     Event for mouse button release
                     '''
                     cb_ax = event.inaxes
                     if cb_ax is not None:
                         cb_ax.press = None
                 def __add_axes(self, ax, size='30%', pad='8%'):
                     '''
                     Add new axes to the given figure
                     '''
                     divider = make_axes_locatable(ax)
                     nax = divider.new_horizontal(size=size, pad=pad)
                     ax.figure.add_axes(nax)
                     return nax
                 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
                     '''
                     Create a masked array for missing data
                     '''
                     if x_buffer.shape[0] < 2:
                         return x_buffer, y_buffer, z_buffer
                     deltas = x_buffer[1:] - x_buffer[0:-1]
                     x_median = numpy.median(deltas)
                     index = numpy.where(deltas > 5 * x_median)
                     if len(index[0]) != 0:
                         z_buffer[::, index[0], ::] = self.__missing
                         z_buffer = numpy.ma.masked_inside(z_buffer,
 .99 * self.__missing,
 .01 * self.__missing)
                     return x_buffer, y_buffer, z_buffer
                 def decimate(self):
                     # dx = int(len(self.x)/self.__MAXNUMX) + 1
                     dy = int(len(self.y) / self.decimation) + 1
                     # x = self.x[::dx]
                     x = self.x
                     y = self.y[::dy]
                     z = self.z[::, ::, ::dy]
                     return x, y, z
                 def format(self):
                     '''
                     Set min and max values, labels, ticks and titles
                     '''
                     if self.xmin is None:
                         xmin = self.data.min_time
                     else:
                         if self.xaxis is 'time':
                             dt = self.getDateTime(self.data.min_time)
                             xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) -
                                     datetime.datetime(1970, 1, 1)).total_seconds()
                             if self.data.localtime:
                                 xmin += time.timezone
                         else:
                             xmin = self.xmin
                     if self.xmax is None:
                         xmax = xmin + self.xrange * 60 * 60
                     else:
                         if self.xaxis is 'time':
                             dt = self.getDateTime(self.data.max_time)
                             xmax = (dt.replace(hour=int(self.xmax), minute=59, second=59) -
                                     datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=1)).total_seconds()
                             if self.data.localtime:
                                 xmax += time.timezone
                         else:
                             xmax = self.xmax
                     ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
                     ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
                     #Y = numpy.array([1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000])
                     #i = 1 if numpy.where(
                     #    abs(ymax-ymin) <= Y)[0][0] < 0 else numpy.where(abs(ymax-ymin) <= Y)[0][0]
                     #ystep = Y[i] / 10.
                     dig = int(numpy.log10(ymax))
                     if dig == 0:
                         digD = len(str(ymax)) - 2
                         ydec = ymax*(10**digD)
                         dig = int(numpy.log10(ydec))
                         ystep = ((ydec + (10**(dig)))//10**(dig))*(10**(dig))
                         ystep = ystep/5
                         ystep = ystep/(10**digD)
                     else:
                         ystep = ((ymax + (10**(dig)))//10**(dig))*(10**(dig))
                         ystep = ystep/5
                     if self.xaxis is not 'time':
                         dig = int(numpy.log10(xmax))
                         if dig <= 0:
                             digD = len(str(xmax)) - 2
                             xdec = xmax*(10**digD)
                             dig = int(numpy.log10(xdec))
                             xstep = ((xdec + (10**(dig)))//10**(dig))*(10**(dig))
                             xstep = xstep*0.5
                             xstep = xstep/(10**digD)
                         else:
                             xstep = ((xmax + (10**(dig)))//10**(dig))*(10**(dig))
                             xstep = xstep/5
                     for n, ax in enumerate(self.axes):
                         if ax.firsttime:
                             ax.set_facecolor(self.bgcolor)
                             ax.yaxis.set_major_locator(MultipleLocator(ystep))
                             if self.xscale:
                                 ax.xaxis.set_major_formatter(FuncFormatter(
                                     lambda x, pos: '{0:g}'.format(x*self.xscale)))
                             if self.xscale:
                                 ax.yaxis.set_major_formatter(FuncFormatter(
                                     lambda x, pos: '{0:g}'.format(x*self.yscale)))
                             if self.xaxis is 'time':
                                 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
                                 ax.xaxis.set_major_locator(LinearLocator(9))
                             else:
                                 ax.xaxis.set_major_locator(MultipleLocator(xstep))
                             if self.xlabel is not None:
                                 ax.set_xlabel(self.xlabel)
                             ax.set_ylabel(self.ylabel)
                             ax.firsttime = False
                             if self.showprofile:
                                 self.pf_axes[n].set_ylim(ymin, ymax)
                                 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
                                 self.pf_axes[n].set_xlabel('dB')
                                 self.pf_axes[n].grid(b=True, axis='x')
                                 [tick.set_visible(False)
                                  for tick in self.pf_axes[n].get_yticklabels()]
                             if self.colorbar:
                                 ax.cbar = plt.colorbar(
                                     ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
                                 ax.cbar.ax.tick_params(labelsize=8)
                                 ax.cbar.ax.press = None
                                 if self.cb_label:
                                     ax.cbar.set_label(self.cb_label, size=8)
                                 elif self.cb_labels:
                                     ax.cbar.set_label(self.cb_labels[n], size=8)
                             else:
                                 ax.cbar = None
                         if self.grid:
                             ax.grid(True)
                         if not self.polar:
                             ax.set_xlim(xmin, xmax)
                             ax.set_ylim(ymin, ymax)
                             ax.set_title('{} {} {}'.format(
                                 self.titles[n],
                                 self.getDateTime(self.data.max_time).strftime(
                                     '%Y-%m-%d %H:%M:%S'),
                                 self.time_label),
                                 size=8)
                         else:
                             ax.set_title('{}'.format(self.titles[n]), size=8)
                             ax.set_ylim(0, 90)
                             ax.set_yticks(numpy.arange(0, 90, 20))
                             ax.yaxis.labelpad = 40
                 def clear_figures(self):
                     '''
                     Reset axes for redraw plots
                     '''
                     for ax in self.axes:
                         ax.clear()
                         ax.firsttime = True
                         if ax.cbar:
                             ax.cbar.remove()
                 def __plot(self):
                     '''
                     Main function to plot, format and save figures
                     '''
                     try:
                         self.plot()
                         self.format()
                     except Exception as e:
                        log.warning('{} Plot could not be updated... check data'.format(
                            self.CODE), self.name)
                        log.error(str(e), '')
                        return
                     for n, fig in enumerate(self.figures):
                         if self.nrows == 0 or self.nplots == 0:
                             log.warning('No data', self.name)
                             fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
                             fig.canvas.manager.set_window_title(self.CODE)
                             continue
                         fig.tight_layout()
                         fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
                                                                              self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
                         fig.canvas.draw()
                         if self.show:
                             fig.show()
                             figpause(0.1)
                         if self.save:
                             self.save_figure(n)
                     if self.plot_server:
                         self.send_to_server()
                         # t = Thread(target=self.send_to_server)
                         # t.start()
                 def save_figure(self, n):
                     '''
                     '''
                     if self.save_counter < self.save_period:
                         self.save_counter += 1
                         return
                     self.save_counter = 1
                     fig = self.figures[n]
                     if self.save_labels:
                         labels = self.save_labels
                     else:
                         labels = list(range(self.nrows))
                     if self.oneFigure:
                         label = ''
                     else:
                         label = '-{}'.format(labels[n])
                     figname = os.path.join(
                         self.save,
                         self.CODE,
                         '{}{}_{}.png'.format(
                             self.CODE,
                             label,
-                            self.getDateTime(self.data.max_time).strftime(
+                            self.getDateTime(self.data.max_time).strftime('%Y%m%d_%H%M%S'),
-                                '%Y%m%d_%H%M%S'
-                                ),
                             )
                         )
                     log.log('Saving figure: {}'.format(figname), self.name)
                     if not os.path.isdir(os.path.dirname(figname)):
                         os.makedirs(os.path.dirname(figname))
                     fig.savefig(figname)
                     if self.realtime:
                         figname = os.path.join(
                             self.save,
                             '{}{}_{}.png'.format(
                                 self.CODE,
                                 label,
                                 self.getDateTime(self.data.min_time).strftime(
                                     '%Y%m%d'
                                     ),
                                 )
                             )
                         fig.savefig(figname)
                 def send_to_server(self):
                     '''
                     '''
                     if self.sender_counter < self.sender_period:
                         self.sender_counter += 1
                         return
                     self.sender_counter = 1
                     self.data.meta['titles'] = self.titles
                     retries = 2
                     while True:
                         self.socket.send_string(self.data.jsonify(self.plot_name, self.plot_type))
                         socks = dict(self.poll.poll(5000))
                         if socks.get(self.socket) == zmq.POLLIN:
                             reply = self.socket.recv_string()
                             if reply == 'ok':
                                 log.log("Response from server ok", self.name)
                                 break
                             else:
                                 log.warning(
                                     "Malformed reply from server: {}".format(reply), self.name)
                         else:
                             log.warning(
                                 "No response from server, retrying...", self.name)
                         self.socket.setsockopt(zmq.LINGER, 0)
                         self.socket.close()
                         self.poll.unregister(self.socket)
                         retries -= 1
                         if retries == 0:
                             log.error(
                                 "Server seems to be offline, abandoning", self.name)
                             self.socket = self.context.socket(zmq.REQ)
                             self.socket.connect(self.plot_server)
                             self.poll.register(self.socket, zmq.POLLIN)
                             time.sleep(1)
                             break
                         self.socket = self.context.socket(zmq.REQ)
                         self.socket.connect(self.plot_server)
                         self.poll.register(self.socket, zmq.POLLIN)
                         time.sleep(0.5)
                 def setup(self):
                     '''
                     This method should be implemented in the child class, the following
                     attributes should be set:
                     self.nrows: number of rows
                     self.ncols: number of cols
                     self.nplots: number of plots (channels or pairs)
                     self.ylabel: label for Y axes
                     self.titles: list of axes title
                     '''
                     raise NotImplementedError
                 def plot(self):
                     '''
                     Must be defined in the child class
                     '''
                     raise NotImplementedError
                 def run(self, dataOut, **kwargs):
                     '''
                     Main plotting routine
                     '''
                     if self.isConfig is False:
                         self.__setup(**kwargs)
                         if dataOut.type == 'Parameters':
                             t = dataOut.utctimeInit
                         else:
                             t = dataOut.utctime
                         if dataOut.useLocalTime:
                             self.getDateTime = datetime.datetime.fromtimestamp
                             if not self.localtime:
                                 t += time.timezone
                         else:
                             self.getDateTime = datetime.datetime.utcfromtimestamp
                             if self.localtime:
                                 t -= time.timezone
                         if 'buffer' in self.plot_type:
                             if self.xmin is None:
                                 self.tmin = t
                             else:
                                 self.tmin = (
                                     self.getDateTime(t).replace(
                                         hour=self.xmin,
                                         minute=0,
                                         second=0) - self.getDateTime(0)).total_seconds()
                         self.data.setup()
                         self.isConfig = True
                         if self.plot_server:
                             self.context = zmq.Context()
                             self.socket = self.context.socket(zmq.REQ)
                             self.socket.connect(self.plot_server)
                             self.poll = zmq.Poller()
                             self.poll.register(self.socket, zmq.POLLIN)
                     if dataOut.type == 'Parameters':
                         tm = dataOut.utctimeInit
                     else:
                         tm = dataOut.utctime
                     if not dataOut.useLocalTime and self.localtime:
                         tm -= time.timezone
                     if dataOut.useLocalTime and not self.localtime:
                         tm += time.timezone
                     if self.xaxis is 'time' and self.data and (tm - self.tmin) >= self.xrange*60*60:
                         self.save_counter = self.save_period
                         self.__plot()
                         self.xmin += self.xrange
                         if self.xmin >= 24:
                             self.xmin -= 24
                         self.tmin += self.xrange*60*60
                         self.data.setup()
                         self.clear_figures()
                     self.data.update(dataOut, tm)
                     if self.isPlotConfig is False:
                         self.__setup_plot()
                         self.isPlotConfig = True
                     if self.realtime:
                         self.__plot()
                     else:
                         self.__throttle_plot(self.__plot)#, coerce=coerce)
                 def close(self):
                     if self.data:
                         self.save_counter = self.save_period
                         self.__plot()
                     if self.data and self.pause:
                         figpause(10)

schainpy/model/io/jroIO_kamisr.py +37 -17

             '''
             Created on Set 9, 2015
             @author: roj-idl71 Karim Kuyeng
             '''
             import os
             import sys
             import glob
             import fnmatch
             import datetime
             import time
             import re
             import h5py
             import numpy
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
-            from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
+            from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from numpy import imag
+            @MPDecorator
             class AMISRReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self)
                     self.set = None
                     self.subset = None
                     self.extension_file = '.h5'
                     self.dtc_str = 'dtc'
                     self.dtc_id = 0
                     self.status = True
                     self.isConfig = False
                     self.dirnameList = []
                     self.filenameList = []
                     self.fileIndex = None
                     self.flagNoMoreFiles = False
                     self.flagIsNewFile = 0
                     self.filename = ''
                     self.amisrFilePointer = None
-                    self.dataset = None
+                    #self.dataset = None
                     self.profileIndex = 0
                     self.beamCodeByFrame = None
                     self.radacTimeByFrame = None
                     self.dataset = None
                     self.__firstFile = True
                     self.buffer = None
                     self.timezone = 'ut'
                     self.__waitForNewFile = 20
                     self.__filename_online = None
                     #Is really necessary create the output object in the initializer
                     self.dataOut = Voltage()
+                    self.dataOut.error=False
                 def setup(self,path=None,
                                 startDate=None,
                                 endDate=None,
                                 startTime=None,
                                 endTime=None,
                                 walk=True,
                                 timezone='ut',
                                 all=0,
                                 code = None,
                                 nCode = 0,
                                 nBaud = 0,
                                 online=False):
+                    #print ("T",path)
                     self.timezone = timezone
                     self.all = all
                     self.online = online
                     self.code = code
                     self.nCode = int(nCode)
                     self.nBaud = int(nBaud)
                     #self.findFiles()
                     if not(online):
                         #Busqueda de archivos offline
                         self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
                     else:
                         self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
                     if not(self.filenameList):
                         print("There is no files into the folder: %s"%(path))
                         sys.exit(-1)
                     self.fileIndex = -1
                     self.readNextFile(online)
                     '''
                     Add code
                     '''
                     self.isConfig = True
                     pass
                 def readAMISRHeader(self,fp):
                     header = 'Raw11/Data/RadacHeader'
                     self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
                     self.beamCode = fp.get('Raw11/Data/Beamcodes') # NUMBER OF CHANNELS AND IDENTIFY POSITION TO CREATE A FILE WITH THAT INFO
                     #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
                     self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
                     self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
                     self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
                     self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
                     self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
                     self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
                     self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
                     self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
                     self.frequency =  fp.get('Rx/Frequency')
                     txAus = fp.get('Raw11/Data/Pulsewidth')
                     self.nblocks = self.pulseCount.shape[0] #nblocks
                     self.nprofiles = self.pulseCount.shape[1] #nprofile
                     self.nsa = self.nsamplesPulse[0,0] #ngates
                     self.nchannels = self.beamCode.shape[1]
                     self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
                     #self.__waitForNewFile = self.nblocks  # wait depending on the number of blocks since each block is 1 sec
                     self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
                     #filling radar controller header parameters
                     self.__ippKm = self.ippSeconds *.15*1e6 # in km
                     self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
                     self.__txB = 0
                     nWindows=1
                     self.__nSamples = self.nsa
                     self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
                     self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
                     #for now until understand why the code saved is different (code included even though code not in tuf file)
                     #self.__codeType = 0
                    # self.__nCode = None
                    # self.__nBaud = None
                     self.__code = self.code
                     self.__codeType = 0
                     if self.code != None:
                         self.__codeType = 1
                     self.__nCode = self.nCode
                     self.__nBaud = self.nBaud
                     #self.__code = 0
                     #filling system header parameters
                     self.__nSamples = self.nsa
                     self.newProfiles = self.nprofiles/self.nchannels
                     self.__channelList = list(range(self.nchannels))
                     self.__frequency = self.frequency[0][0]
                 def createBuffers(self):
                     pass
                 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.walk = walk
                 def __checkPath(self):
                     if os.path.exists(self.path):
                         self.status = 1
                     else:
                         self.status = 0
                         print('Path:%s does not exists'%self.path)
                     return
                 def __selDates(self, amisr_dirname_format):
                     try:
                         year = int(amisr_dirname_format[0:4])
                         month = int(amisr_dirname_format[4:6])
                         dom = int(amisr_dirname_format[6:8])
                         thisDate = datetime.date(year,month,dom)
                         if (thisDate>=self.startDate and thisDate <= self.endDate):
                             return amisr_dirname_format
                     except:
                         return None
                 def __findDataForDates(self,online=False):
                     if not(self.status):
                         return None
                     pat = '\d+.\d+'
                     dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
                     dirnameList = [x for x in dirnameList if x!=None]
                     dirnameList = [x.string for x in dirnameList]
                     if not(online):
                         dirnameList = [self.__selDates(x) for x in dirnameList]
                         dirnameList = [x for x in dirnameList if x!=None]
                     if len(dirnameList)>0:
                         self.status = 1
                         self.dirnameList = dirnameList
                         self.dirnameList.sort()
                     else:
                         self.status = 0
                         return None
                 def __getTimeFromData(self):
                     startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
                     endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                     print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
                     print('........................................')
                     filter_filenameList = []
                     self.filenameList.sort()
                     #for i in range(len(self.filenameList)-1):
                     for i in range(len(self.filenameList)):
                         filename = self.filenameList[i]
                         fp = h5py.File(filename,'r')
                         time_str = fp.get('Time/RadacTimeString')
-                        startDateTimeStr_File = time_str[0][0].split('.')[0]
+                        startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
+                        #startDateTimeStr_File = "2019-12-16 09:21:11"
                         junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                         startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
-                        endDateTimeStr_File = time_str[-1][-1].split('.')[0]
+                        #endDateTimeStr_File = "2019-12-16 11:10:11"
+                        endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
                         junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                         endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                         fp.close()
+                        #print("check time", startDateTime_File)
                         if self.timezone == 'lt':
                             startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
                             endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
                         if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
                             #self.filenameList.remove(filename)
                             filter_filenameList.append(filename)
                         if (endDateTime_File>=endDateTime_Reader):
                             break
                     filter_filenameList.sort()
                     self.filenameList = filter_filenameList
                     return 1
                 def __filterByGlob1(self, dirName):
                     filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
                     filter_files.sort()
                     filterDict = {}
                     filterDict.setdefault(dirName)
                     filterDict[dirName] = filter_files
                     return filterDict
                 def __getFilenameList(self, fileListInKeys, dirList):
                     for value in fileListInKeys:
                         dirName = list(value.keys())[0]
                         for file in value[dirName]:
                             filename = os.path.join(dirName, file)
                             self.filenameList.append(filename)
                 def __selectDataForTimes(self, online=False):
                     #aun no esta implementado el filtro for tiempo
                     if not(self.status):
                         return None
                     dirList = [os.path.join(self.path,x) for x in self.dirnameList]
                     fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
                     self.__getFilenameList(fileListInKeys, dirList)
                     if not(online):
                         #filtro por tiempo
                         if not(self.all):
                             self.__getTimeFromData()
                         if len(self.filenameList)>0:
                             self.status = 1
                             self.filenameList.sort()
                         else:
                             self.status = 0
                             return None
                     else:
                         #get the last file - 1
                         self.filenameList = [self.filenameList[-2]]
                     new_dirnameList = []
                     for dirname in self.dirnameList:
                         junk = numpy.array([dirname in x for x in self.filenameList])
                         junk_sum = junk.sum()
                         if junk_sum > 0:
                             new_dirnameList.append(dirname)
                     self.dirnameList = new_dirnameList
                     return 1
                 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),walk=True):
                     if endDate ==None:
                      startDate = datetime.datetime.utcnow().date()
                      endDate = datetime.datetime.utcnow().date()
                     self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
                     self.__checkPath()
                     self.__findDataForDates(online=True)
                     self.dirnameList = [self.dirnameList[-1]]
                     self.__selectDataForTimes(online=True)
                     return
                 def searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         walk=True):
                     self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
                     self.__checkPath()
                     self.__findDataForDates()
                     self.__selectDataForTimes()
                     for i in range(len(self.filenameList)):
                         print("%s" %(self.filenameList[i]))
                     return
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print("No more Files")
+                            self.dataOut.error = True
                             return 0
                         filename = self.filenameList[idFile]
                         amisrFilePointer = h5py.File(filename,'r')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.amisrFilePointer = amisrFilePointer
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def __setNextFileOnline(self):
                     filename = self.filenameList[0]
                     if self.__filename_online != None:
                         self.__selectDataForTimes(online=True)
                         filename = self.filenameList[0]
                         wait = 0
                         while self.__filename_online == filename:
                             print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
                             if wait == 5:
                                 return 0
                             sleep(self.__waitForNewFile)
                             self.__selectDataForTimes(online=True)
                             filename = self.filenameList[0]
                             wait += 1
                     self.__filename_online = filename
                     self.amisrFilePointer = h5py.File(filename,'r')
                     self.flagIsNewFile = 1
                     self.filename = filename
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def readData(self):
                     buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
                     re = buffer[:,:,:,0]
                     im = buffer[:,:,:,1]
                     dataset = re + im*1j
                     self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
                     timeset = self.radacTime[:,0]
                     return dataset,timeset
                 def reshapeData(self):
                 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
                     channels = self.beamCodeByPulse[0,:]
                     nchan = self.nchannels
                     #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
                     nblocks = self.nblocks
                     nsamples = self.nsa
                     #Dimensions : nChannels, nProfiles, nSamples
-                    new_block = numpy.empty((nblocks, nchan, self.newProfiles, nsamples), dtype="complex64")
+                    new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
                     ############################################
                     for thisChannel in range(nchan):
                         new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[0][thisChannel])[0],:]
                     new_block = numpy.transpose(new_block, (1,0,2,3))
                     new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
                     return new_block
                 def updateIndexes(self):
                     pass
                 def fillJROHeader(self):
                     #fill radar controller header
-                    self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
+                    self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
                                                                                   txA=self.__txA,
                                                                                   txB=0,
                                                                                   nWindows=1,
                                                                                   nHeights=self.__nSamples,
                                                                                   firstHeight=self.__firstHeight,
                                                                                   deltaHeight=self.__deltaHeight,
                                                                                   codeType=self.__codeType,
                                                                                   nCode=self.__nCode, nBaud=self.__nBaud,
                                                                                   code = self.__code,
                                                                                   fClock=1)
                     #fill system header
                     self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                 nProfiles=self.newProfiles,
                                                                 nChannels=len(self.__channelList),
                                                                 adcResolution=14,
-                                                                pciDioBusWith=32)
+                                                                pciDioBusWidth=32)
                     self.dataOut.type = "Voltage"
                     self.dataOut.data = None
                     self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
             #        self.dataOut.nChannels = 0
             #        self.dataOut.nHeights = 0
                     self.dataOut.nProfiles = self.newProfiles*self.nblocks
                     #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
                     ranges = numpy.reshape(self.rangeFromFile.value,(-1))
                     self.dataOut.heightList =  ranges/1000.0 #km
                     self.dataOut.channelList = self.__channelList
                     self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
             #        self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData = True
                     #Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime = None
                     #self.dataOut.timeZone = -5 #self.__timezone/60  #timezone like jroheader, difference in minutes between UTC and localtime
                     if self.timezone == 'lt':
                         self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
                     else:
                         self.dataOut.timeZone = 0 #by default time is UTC
                     self.dataOut.dstFlag = 0
                     self.dataOut.errorCount = 0
                     self.dataOut.nCohInt = 1
                     self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
                     self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.ippSeconds = self.ippSeconds
                     #Time interval between profiles
                     #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.__frequency
                     self.dataOut.realtime = self.online
                     pass
                 def readNextFile(self,online=False):
                     if not(online):
                         newFile = self.__setNextFileOffline()
                     else:
                         newFile = self.__setNextFileOnline()
                     if not(newFile):
                         return 0
                     #if self.__firstFile:
                     self.readAMISRHeader(self.amisrFilePointer)
                     self.createBuffers()
                     self.fillJROHeader()
                     #self.__firstFile = False
                     self.dataset,self.timeset = self.readData()
                     if self.endDate!=None:
                      endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                      time_str = self.amisrFilePointer.get('Time/RadacTimeString')
-                     startDateTimeStr_File = time_str[0][0].split('.')[0]
+                     startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
                      junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                      startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                      if self.timezone == 'lt':
                       startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
                      if (startDateTime_File>endDateTime_Reader):
                          return 0
                     self.jrodataset = self.reshapeData()
                     #----self.updateIndexes()
                     self.profileIndex = 0
                     return 1
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= (self.newProfiles*self.nblocks):
                         return 1
                     return 0
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         return 0
                     if self.__hasNotDataInBuffer():
                         if not (self.readNextFile(self.online)):
                             return 0
-                    if self.dataset is None: # setear esta condicion cuando no hayan datos por leers
+                    if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
                         self.dataOut.flagNoData = True
                         return 0
                     #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
                     self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
+                    #print("R_t",self.timeset)
                     #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
                     #verificar basic header de jro data y ver si es compatible con este valor
                     #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
                     indexprof = numpy.mod(self.profileIndex, self.newProfiles)
                     indexblock = self.profileIndex/self.newProfiles
-                    #print indexblock, indexprof
+                    #print (indexblock, indexprof)
-                    self.dataOut.utctime = self.timeset[indexblock] + (indexprof * self.ippSeconds * self.nchannels)
+                    diffUTC = 1.8e4 #UTC diference from peru in seconds --Joab
+                    diffUTC = 0
+                    t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
+                    #cambio posible 18/02/2020
+                    #print("utc :",indexblock," __ ",t_comp)
+                    #print(numpy.shape(self.timeset))
+                    self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
+                    #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
+                    #print(self.dataOut.utctime)
                     self.dataOut.profileIndex = self.profileIndex
                     self.dataOut.flagNoData = False
             #         if indexprof == 0:
             #             print self.dataOut.utctime
                     self.profileIndex += 1
                     return self.dataOut.data
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()

schainpy/model/io/jroIO_param.py +1 -1

             import numpy
             import time
             import os
             import h5py
             import re
             import datetime
             import schainpy.admin
             from schainpy.model.data.jrodata import *
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.io.jroIO_base import *
             from schainpy.utils import log
             @MPDecorator
             class ParamReader(JRODataReader,ProcessingUnit):
                 '''
                 Reads HDF5 format files
                 path
                 startDate
                 endDate
                 startTime
                 endTime
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 timezone = None
                 startTime = None
                 endTime = None
                 fileIndex = None
                 utcList = None      #To select data in the utctime list
                 blockList = None    #List to blocks to be read from the file
                 blocksPerFile = None    #Number of blocks to be read
                 blockIndex = None
                 path = None
                 #List of Files
                 filenameList = None
                 datetimeList = None
                 #Hdf5 File
                 listMetaname = None
                 listMeta = None
                 listDataname = None
                 listData = None
                 listShapes = None
                 fp = None
                 #dataOut reconstruction
                 dataOut = None
                 def __init__(self):#, **kwargs):
                     ProcessingUnit.__init__(self) #, **kwargs)
                     self.dataOut = Parameters()
                     return
                 def setup(self, **kwargs):
                     path = kwargs['path']
                     startDate = kwargs['startDate']
                     endDate = kwargs['endDate']
                     startTime = kwargs['startTime']
                     endTime = kwargs['endTime']
                     walk = kwargs['walk']
                     if 'ext' in kwargs:
                         ext = kwargs['ext']
                     else:
                         ext = '.hdf5'
                     if 'timezone' in kwargs:
                         self.timezone = kwargs['timezone']
                     else:
                         self.timezone = 'lt'
                     print("[Reading] Searching files in offline mode ...")
                     pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            ext=ext, walk=walk)
                     if not(filenameList):
                         print("There is no files into the folder: %s"%(path))
                         sys.exit(-1)
                     self.fileIndex = -1
                     self.startTime = startTime
                     self.endTime = endTime
                     self.__readMetadata()
                     self.__setNextFileOffline()
                     return
                 def searchFilesOffLine(self,
                                         path,
                                         startDate=None,
                                         endDate=None,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         ext='.hdf5',
                                         walk=True):
                     expLabel = ''
                     self.filenameList = []
                     self.datetimeList = []
                     pathList = []
                     JRODataObj = JRODataReader()
                     dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
                     if dateList == []:
                         print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime()))
                         return None, None
                     if len(dateList) > 1:
                         print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
                     else:
                         print("[Reading] data was found for the date %s" %(dateList[0]))
                     filenameList = []
                     datetimeList = []
                     #----------------------------------------------------------------------------------
                     for thisPath in pathList:
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             if not isFileInDateRange(filename, startDate, endDate):
                                 continue
                             thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
                         return None, None
                     print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
                     print()
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
                     """
                     Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                     Inputs:
                         filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                         startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                         endDate            :    fecha final del rango seleccionado en formato datetime.date
                         startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                         endTime            :    tiempo final del rango seleccionado en formato datetime.time
                     Return:
                         Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                         fecha especificado, de lo contrario retorna False.
                     Excepciones:
                         Si el archivo no existe o no puede ser abierto
                         Si la cabecera no puede ser leida.
                     """
                     try:
                         fp = h5py.File(filename,'r')
                         grp1 = fp['Data']
                     except IOError:
                         traceback.print_exc()
                         raise IOError("The file %s can't be opened" %(filename))
                     #In case has utctime attribute
                     grp2 = grp1['utctime']
              #         thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
                     thisUtcTime = grp2.value[0]
                     fp.close()
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
                     #General case
                     #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                     #-----------o----------------------------o-----------
                     #       startTime                     endTime
                     if endTime >= startTime:
                         thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
                         if numpy.any(thisUtcLog):   #If there is one block between the hours mentioned
                             return thisDatetime
                         return None
                     #If endTime < startTime then endTime belongs to the next day
                     #<<<<<<<<<<<o                            o>>>>>>>>>>>
                     #-----------o----------------------------o-----------
                     #        endTime                    startTime
                     if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
                         return None
                     if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
                         return None
                     if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
                         return None
                     return thisDatetime
                 def __setNextFileOffline(self):
                     self.fileIndex += 1
                     idFile = self.fileIndex
                     if not(idFile < len(self.filenameList)):
                         raise schainpy.admin.SchainError("No more Files")
                         return 0
                     filename = self.filenameList[idFile]
                     filePointer = h5py.File(filename,'r')
                     self.filename = filename
                     self.fp = filePointer
                     print("Setting the file: %s"%self.filename)
                     self.__setBlockList()
                     self.__readData()
                     self.blockIndex = 0
                     return 1
                 def __setBlockList(self):
                     '''
                     Selects the data within the times defined
                     self.fp
                     self.startTime
                     self.endTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     fp = self.fp
                     startTime = self.startTime
                     endTime = self.endTime
                     grp = fp['Data']
                     thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
                     #ERROOOOR
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     return
                 def __readMetadata(self):
                     '''
                     Reads Metadata
                     self.pathMeta
                     self.listShapes
                     self.listMetaname
                     self.listMeta
                     '''
                     filename = self.filenameList[0]
                     fp = h5py.File(filename,'r')
                     gp = fp['Metadata']
                     listMetaname = []
                     listMetadata = []
                     for item in list(gp.items()):
                         name = item[0]
                         if name=='array dimensions':
                             table = gp[name][:]
                             listShapes = {}
                             for shapes in table:
                                 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
                         else:
                             data = gp[name].value
                             listMetaname.append(name)
                             listMetadata.append(data)
                     self.listShapes = listShapes
                     self.listMetaname = listMetaname
                     self.listMeta = listMetadata
                     fp.close()
                     return
                 def __readData(self):
                     grp = self.fp['Data']
                     listdataname = []
                     listdata = []
                     for item in list(grp.items()):
                         name = item[0]
                         listdataname.append(name)
                         array = self.__setDataArray(grp[name],self.listShapes[name])
                         listdata.append(array)
                     self.listDataname = listdataname
                     self.listData = listdata
                     return
                 def __setDataArray(self, dataset, shapes):
                     nDims = shapes[0]
                     nDim2 = shapes[1]      #Dimension 0
                     nDim1 = shapes[2]      #Dimension 1, number of Points or Parameters
                     nDim0 = shapes[3]      #Dimension 2, number of samples or ranges
                     mode = shapes[4]        #Mode of storing
                     blockList = self.blockList
                     blocksPerFile = self.blocksPerFile
                     #Depending on what mode the data was stored
                     if mode == 0:       #Divided in channels
                         arrayData = dataset.value.astype(numpy.float)[0][blockList]
                     if mode == 1:     #Divided in parameter
                         strds = 'table'
                         nDatas = nDim1
                         newShapes = (blocksPerFile,nDim2,nDim0)
                     elif mode==2:       #Concatenated in a table
                         strds = 'table0'
                         arrayData = dataset[strds].value
                         #Selecting part of the dataset
                         utctime = arrayData[:,0]
                         u, indices = numpy.unique(utctime, return_index=True)
                         if blockList.size != indices.size:
                             indMin = indices[blockList[0]]
                             if blockList[1] + 1 >= indices.size:
                                 arrayData = arrayData[indMin:,:]
                             else:
                                 indMax = indices[blockList[1] + 1]
                                 arrayData = arrayData[indMin:indMax,:]
                         return arrayData
                     #    One dimension
                     if nDims == 0:
                         arrayData = dataset.value.astype(numpy.float)[0][blockList]
                     #    Two dimensions
                     elif nDims == 2:
                         arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
                         newShapes = (blocksPerFile,nDim0)
                         nDatas = nDim1
                         for i in range(nDatas):
                             data = dataset[strds + str(i)].value
                             arrayData[:,i,:] = data[blockList,:]
                     #    Three dimensions
                     else:
                         arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
                         for i in range(nDatas):
                             data = dataset[strds + str(i)].value
                             for b in range(blockList.size):
                                 arrayData[b,:,i,:] = data[:,:,blockList[b]]
                     return arrayData
                 def __setDataOut(self):
                     listMeta = self.listMeta
                     listMetaname = self.listMetaname
                     listDataname = self.listDataname
                     listData = self.listData
                     listShapes = self.listShapes
                     blockIndex = self.blockIndex
              #         blockList = self.blockList
                     for i in range(len(listMeta)):
                         setattr(self.dataOut,listMetaname[i],listMeta[i])
                     for j in range(len(listData)):
                         nShapes = listShapes[listDataname[j]][0]
                         mode = listShapes[listDataname[j]][4]
                         if nShapes == 1:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
                         elif nShapes > 1:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
                         elif mode==0:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
                         #Mode Meteors
                         elif mode ==2:
                             selectedData = self.__selectDataMode2(listData[j], blockIndex)
                             setattr(self.dataOut, listDataname[j], selectedData)
                     return
                 def __selectDataMode2(self, data, blockIndex):
                     utctime = data[:,0]
                     aux, indices = numpy.unique(utctime, return_inverse=True)
                     selInd = numpy.where(indices == blockIndex)[0]
                     selData = data[selInd,:]
                     return selData
                 def getData(self):
                     if self.blockIndex==self.blocksPerFile:
                          if not( self.__setNextFileOffline() ):
                             self.dataOut.flagNoData = True
                             return 0
                     self.__setDataOut()
                     self.dataOut.flagNoData = False
                     self.blockIndex += 1
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
                     return
             @MPDecorator
             class ParamWriter(Operation):
                 '''
                 HDF5 Writer, stores parameters data in HDF5 format files
                 path:             path where the files will be stored
                 blocksPerFile:    number of blocks that will be saved in per HDF5 format file
                 mode:             selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
                 metadataList:     list of attributes that will be stored as metadata
                 dataList:         list of attributes that will be stores as data
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 metaoptchar = "M"
                 metaFile = None
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 grp = None
                 ds = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 dsList = None   #List of dictionaries with dataset properties
                 tableDim = None
                 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
                 currentDay = None
                 lastTime = None
                 setType = None
                 def __init__(self):
                     Operation.__init__(self)
                     return
                 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataList = dataList
                     self.dataOut = dataOut
                     self.mode = mode
                     if self.mode is not None:
                         self.mode = numpy.zeros(len(self.dataList)) + mode
                     else:
                         self.mode = numpy.ones(len(self.dataList))
                     self.setType = setType
                     arrayDim = numpy.zeros((len(self.dataList),5))
                     #Table dimensions
                     dtype0 = self.dtype
                     tableList = []
                     #Dictionary and list of tables
                     dsList = []
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         dataAux = getattr(self.dataOut, self.dataList[i])
                         dsDict['variable'] = self.dataList[i]
                         #---------------------    Conditionals    ------------------------
                         #There is no data
                         if dataAux is None:
                             return 0
                         if isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
                             dsDict['mode'] = 0
                             dsDict['nDim'] = 0
                             arrayDim[i,0] = 0
                             dsList.append(dsDict)
                         #Mode 2: meteors
                         elif self.mode[i] == 2:
                             dsDict['dsName'] = 'table0'
                             dsDict['mode'] = 2      # Mode meteors
                             dsDict['shape'] = dataAux.shape[-1]
                             dsDict['nDim'] = 0
                             dsDict['dsNumber'] = 1
                             arrayDim[i,3] = dataAux.shape[-1]
                             arrayDim[i,4] = self.mode[i]         #Mode the data was stored
                             dsList.append(dsDict)
                         #Mode 1
                         else:
                             arrayDim0 = dataAux.shape       #Data dimensions
                             arrayDim[i,0] = len(arrayDim0)  #Number of array dimensions
                             arrayDim[i,4] = self.mode[i]         #Mode the data was stored
                             strtable = 'table'
                             dsDict['mode'] = 1      # Mode parameters
                             # Three-dimension arrays
                             if len(arrayDim0) == 3:
                                 arrayDim[i,1:-1] = numpy.array(arrayDim0)
                                 nTables = int(arrayDim[i,2])
                                 dsDict['dsNumber'] = nTables
                                 dsDict['shape'] = arrayDim[i,2:4]
                                 dsDict['nDim'] = 3
                                 for j in range(nTables):
                                     dsDict = dsDict.copy()
                                     dsDict['dsName'] = strtable + str(j)
                                     dsList.append(dsDict)
                             # Two-dimension arrays
                             elif len(arrayDim0) == 2:
                                 arrayDim[i,2:-1] = numpy.array(arrayDim0)
                                 nTables = int(arrayDim[i,2])
                                 dsDict['dsNumber'] = nTables
                                 dsDict['shape'] = arrayDim[i,3]
                                 dsDict['nDim'] = 2
                                 for j in range(nTables):
                                     dsDict = dsDict.copy()
                                     dsDict['dsName'] = strtable + str(j)
                                     dsList.append(dsDict)
                             # One-dimension arrays
                             elif len(arrayDim0) == 1:
                                 arrayDim[i,3] = arrayDim0[0]
                                 dsDict['shape'] = arrayDim0[0]
                                 dsDict['dsNumber'] = 1
                                 dsDict['dsName'] = strtable + str(0)
                                 dsDict['nDim'] = 1
                                 dsList.append(dsDict)
                         table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
                         tableList.append(table)
                     self.dsList = dsList
                     self.tableDim = numpy.array(tableList, dtype = dtype0)
                     self.blockIndex = 0
                     timeTuple = time.localtime(dataOut.utctime)
                     self.currentDay = timeTuple.tm_yday
                 def putMetadata(self):
                     fp = self.createMetadataFile()
                     self.writeMetadata(fp)
                     fp.close()
                     return
                 def createMetadataFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = ''
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         filesList = sorted( filesList, key=str.lower )
                         if len( filesList ) > 0:
                             filesList = [k for k in filesList if k.startswith(self.metaoptchar)]
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     filename = os.path.join( path, subfolder, file )
                     self.metaFile = file
                     #Setting HDF5 File
                     fp = h5py.File(filename,'w')
                     return fp
                 def writeMetadata(self, fp):
                     grp = fp.create_group("Metadata")
                     grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
                     for i in range(len(self.metadataList)):
                         grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
                     return
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     if self.lastTime is None:
                         self.lastTime = currentTime
                     #Day
                     timeTuple = time.localtime(currentTime)
                     dataDay = timeTuple.tm_yday
                     #Time
                     timeDiff = currentTime - self.lastTime
                     #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                     if dataDay != self.currentDay:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > 3*60*60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     mode = self.mode
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if os.path.exists(fullpath):
                         filesList = os.listdir( fullpath )
-                        filesList = [k for k in filesList if 'M' in k]
+                        ##filesList = [k for k in filesList if 'M' in k]
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     filename = os.path.join( path, subfolder, file )
                     #Setting HDF5 File
                     fp = h5py.File(filename,'w')
                     #write metadata
                     self.writeMetadata(fp)
                     #Write data
                     grp = fp.create_group("Data")
                     ds = []
                     data = []
                     dsList = self.dsList
                     i = 0
                     while i < len(dsList):
                         dsInfo = dsList[i]
                         #One-dimension data
                         if dsInfo['mode'] == 0:
                             ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
                             ds.append(ds0)
                             data.append([])
                             i += 1
                             continue
                         elif dsInfo['mode'] == 2:
                             grp0 = grp.create_group(dsInfo['variable'])
                             ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
                             ds.append(ds0)
                             data.append([])
                             i += 1
                             continue
                         elif dsInfo['mode'] == 1:
                             grp0 = grp.create_group(dsInfo['variable'])
                             for j in range(dsInfo['dsNumber']):
                                 dsInfo = dsList[i]
                                 tableName = dsInfo['dsName']
                                 if dsInfo['nDim'] == 3:
                                     shape = dsInfo['shape'].astype(int)
                                     ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
                                 else:
                                     shape = int(dsInfo['shape'])
                                     ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
                                 ds.append(ds0)
                                 data.append([])
                                 i += 1
                     fp.flush()
                     fp.close()
                     log.log('creating file: {}'.format(filename), 'Writing')
                     self.filename = filename
                     self.ds = ds
                     self.data = data
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     if self.blockIndex == self.blocksPerFile or self.timeFlag():
                         self.setNextFile()
                     self.readBlock()
                     self.setBlock()     #Prepare data to be written
                     self.writeBlock()   #Write data
                     return
                 def readBlock(self):
                     '''
                     data Array configured
                     self.data
                     '''
                     dsList = self.dsList
                     ds = self.ds
                             #Setting HDF5 File
                     fp = h5py.File(self.filename,'r+')
                     grp = fp["Data"]
                     ind = 0
                     while ind < len(dsList):
                         dsInfo = dsList[ind]
                         if dsInfo['mode'] == 0:
                             ds0 = grp[dsInfo['variable']]
                             ds[ind] = ds0
                             ind += 1
                         else:
                             grp0 = grp[dsInfo['variable']]
                             for j in range(dsInfo['dsNumber']):
                                 dsInfo = dsList[ind]
                                 ds0 = grp0[dsInfo['dsName']]
                                 ds[ind] = ds0
                                 ind += 1
                     self.fp = fp
                     self.grp = grp
                     self.ds = ds
                     return
                 def setBlock(self):
                     '''
                     data Array configured
                     self.data
                     '''
                     #Creating Arrays
                     dsList = self.dsList
                     data = self.data
                     ind = 0
                     while ind < len(dsList):
                         dsInfo = dsList[ind]
                         dataAux = getattr(self.dataOut, dsInfo['variable'])
                         mode = dsInfo['mode']
                         nDim = dsInfo['nDim']
                         if mode == 0 or mode == 2 or nDim == 1:
                             data[ind] = dataAux
                             ind += 1
              #             elif nDim == 1:
              #                 data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
              #                 ind += 1
                         elif nDim == 2:
                             for j in range(dsInfo['dsNumber']):
                                 data[ind] = dataAux[j,:]
                                 ind += 1
                         elif nDim == 3:
                             for j in range(dsInfo['dsNumber']):
                                 data[ind] = dataAux[:,j,:]
                                 ind += 1
                     self.data = data
                     return
                 def writeBlock(self):
                     '''
                     Saves the block in the HDF5 file
                     '''
                     dsList = self.dsList
                     for i in range(len(self.ds)):
                         dsInfo = dsList[i]
                         nDim = dsInfo['nDim']
                         mode = dsInfo['mode']
                         #    First time
                         if self.firsttime:
                             if type(self.data[i]) == numpy.ndarray:
                                 if nDim == 3:
                                     self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
                                     self.ds[i].resize(self.data[i].shape)
                                 if mode == 2:
                                     self.ds[i].resize(self.data[i].shape)
                             self.ds[i][:] = self.data[i]
                         else:
                         #    From second time
                             #    Meteors!
                             if mode == 2:
                                 dataShape = self.data[i].shape
                                 dsShape = self.ds[i].shape
                                 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
                                 self.ds[i][dsShape[0]:,:] = self.data[i]
                             #    No dimension
                             elif mode == 0:
                                 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
                                 self.ds[i][0,-1] = self.data[i]
                             #    One dimension
                             elif nDim == 1:
                                 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
                                 self.ds[i][-1,:] = self.data[i]
                             #    Two dimension
                             elif nDim == 2:
                                 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
                                 self.ds[i][self.blockIndex,:] = self.data[i]
                             #    Three dimensions
                             elif nDim == 3:
                                 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
                                 self.ds[i][:,:,-1] = self.data[i]
                     self.firsttime = False
                     self.blockIndex += 1
                     #Close to save changes
                     self.fp.flush()
                     self.fp.close()
                     return
                 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
                     self.dataOut = dataOut
                     if not(self.isConfig):
                         self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList, mode=mode,
                                    setType=setType)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return
             @MPDecorator
             class ParameterReader(Reader, ProcessingUnit):
                 '''
                 Reads HDF5 format files
                 '''
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                     self.ext = ".hdf5"
                     self.optchar = "D"
                     self.timezone = "lt"
                     self.listMetaname = []
                     self.listMeta = []
                     self.listDataname = []
                     self.listData = []
                     self.listShapes = []
                     self.open_file = h5py.File
                     self.open_mode = 'r'
                     self.metadata = False
                     self.filefmt = "*%Y%j***"
                     self.folderfmt = "*%Y%j"
                 def setup(self, **kwargs):
                     self.set_kwargs(**kwargs)
                     if not self.ext.startswith('.'):
                         self.ext = '.{}'.format(self.ext)
                     if self.online:
                         log.log("Searching files in online mode...", self.name)
                         for nTries in range(self.nTries):
                             fullpath = self.searchFilesOnLine(self.path, self.startDate,
                                 self.endDate, self.expLabel, self.ext, self.walk,
                                 self.filefmt, self.folderfmt)
                             try:
                                 fullpath = next(fullpath)
                             except:
                                 fullpath = None
                             if fullpath:
                                 break
                             log.warning(
                                 'Waiting {} sec for a valid file in {}: try {} ...'.format(
                                     self.delay, self.path, nTries + 1),
                                 self.name)
                             time.sleep(self.delay)
                         if not(fullpath):
                             raise schainpy.admin.SchainError(
                                 'There isn\'t any valid file in {}'.format(self.path))
                         pathname, filename = os.path.split(fullpath)
                         self.year = int(filename[1:5])
                         self.doy = int(filename[5:8])
                         self.set = int(filename[8:11]) - 1
                     else:
                         log.log("Searching files in {}".format(self.path), self.name)
                         self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
                             self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
                     self.setNextFile()
                     return
                 def readFirstHeader(self):
                     '''Read metadata and data'''
                     self.__readMetadata()
                     self.__readData()
                     self.__setBlockList()
                     self.blockIndex = 0
                     return
                 def __setBlockList(self):
                     '''
                     Selects the data within the times defined
                     self.fp
                     self.startTime
                     self.endTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     startTime = self.startTime
                     endTime = self.endTime
                     index = self.listDataname.index('utctime')
                     thisUtcTime = self.listData[index]
                     self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     return
                 def __readMetadata(self):
                     '''
                     Reads Metadata
                     '''
                     listMetaname = []
                     listMetadata = []
                     if 'Metadata' in self.fp:
                         gp = self.fp['Metadata']
                         for item in list(gp.items()):
                             name = item[0]
                             if name=='variables':
                                 table = gp[name][:]
                                 listShapes = {}
                                 for shapes in table:
                                     listShapes[shapes[0].decode()] = numpy.array([shapes[1]])
                             else:
                                 data = gp[name].value
                                 listMetaname.append(name)
                                 listMetadata.append(data)
                     elif self.metadata:
                         metadata = json.loads(self.metadata)
                         listShapes = {}
                         for tup in metadata:
                             name, values, dim = tup
                             if dim == -1:
                                 listMetaname.append(name)
                                 listMetadata.append(self.fp[values].value)
                             else:
                                 listShapes[name] = numpy.array([dim])
                     else:
                         raise IOError('Missing Metadata group in file or metadata info')
                     self.listShapes = listShapes
                     self.listMetaname = listMetaname
                     self.listMeta = listMetadata
                     return
                 def __readData(self):
                     listdataname = []
                     listdata = []
                     if 'Data' in self.fp:
                         grp = self.fp['Data']
                         for item in list(grp.items()):
                             name = item[0]
                             listdataname.append(name)
                             dim = self.listShapes[name][0]
                             if dim == 0:
                                 array = grp[name].value
                             else:
                                 array = []
                                 for i in range(dim):
                                     array.append(grp[name]['table{:02d}'.format(i)].value)
                                 array = numpy.array(array)
                             listdata.append(array)
                     elif self.metadata:
                         metadata = json.loads(self.metadata)
                         for tup in metadata:
                             name, values, dim = tup
                             listdataname.append(name)
                             if dim == -1:
                                 continue
                             elif dim == 0:
                                 array = self.fp[values].value
                             else:
                                 array = []
                                 for var in values:
                                     array.append(self.fp[var].value)
                                 array = numpy.array(array)
                             listdata.append(array)
                     else:
                         raise IOError('Missing Data group in file or metadata info')
                     self.listDataname = listdataname
                     self.listData = listdata
                     return
                 def getData(self):
                     for i in range(len(self.listMeta)):
                         setattr(self.dataOut, self.listMetaname[i], self.listMeta[i])
                     for j in range(len(self.listData)):
                         dim = self.listShapes[self.listDataname[j]][0]
                         if dim == 0:
                             setattr(self.dataOut, self.listDataname[j], self.listData[j][self.blockIndex])
                         else:
                             setattr(self.dataOut, self.listDataname[j], self.listData[j][:,self.blockIndex])
                     self.dataOut.paramInterval = self.interval
                     self.dataOut.flagNoData = False
                     self.blockIndex += 1
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     if self.blockIndex == self.blocksPerFile:
                         self.setNextFile()
                     self.getData()
                     return
             @MPDecorator
             class ParameterWriter(Operation):
                 '''
                 HDF5 Writer, stores parameters data in HDF5 format files
                 path:             path where the files will be stored
                 blocksPerFile:    number of blocks that will be saved in per HDF5 format file
                 mode:             selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
                 metadataList:     list of attributes that will be stored as metadata
                 dataList:         list of attributes that will be stores as data
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 metaoptchar = "M"
                 metaFile = None
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 grp = None
                 ds = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 dsList = None   #List of dictionaries with dataset properties
                 tableDim = None
                 dtype = [('name', 'S20'),('nDim', 'i')]
                 currentDay = None
                 lastTime = None
                 def __init__(self):
                     Operation.__init__(self)
                     return
                 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataList = dataList
                     self.setType = setType
                     tableList = []
                     dsList = []
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         dataAux = getattr(self.dataOut, self.dataList[i])
                         dsDict['variable'] = self.dataList[i]
                         if dataAux is None:
                             continue
                         elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
                             dsDict['nDim'] = 0
                         else:
                             dsDict['nDim'] = len(dataAux.shape)
                             dsDict['shape'] = dataAux.shape
                             dsDict['dsNumber'] = dataAux.shape[0]
                         dsList.append(dsDict)
                         tableList.append((self.dataList[i], dsDict['nDim']))
                     self.dsList = dsList
                     self.tableDim = numpy.array(tableList, dtype=self.dtype)
                     self.currentDay = self.dataOut.datatime.date()
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     timeTuple = time.localtime(currentTime)
                     dataDay = timeTuple.tm_yday
                     if self.lastTime is None:
                         self.lastTime = currentTime
                         self.currentDay = dataDay
                         return False
                     timeDiff = currentTime - self.lastTime
                     #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                     if dataDay != self.currentDay:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > 3*60*60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, setType=None):
                     self.dataOut = dataOut
                     if not(self.isConfig):
                         self.setup(path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList,
                                    setType=setType)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join(path, subfolder)
                     if os.path.exists(fullpath):
                         filesList = os.listdir(fullpath)
                         filesList = [k for k in filesList if k.startswith(self.optchar)]
                         if len( filesList ) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     self.filename = os.path.join( path, subfolder, file )
                     #Setting HDF5 File
                     self.fp = h5py.File(self.filename, 'w')
                     #write metadata
                     self.writeMetadata(self.fp)
                     #Write data
                     self.writeData(self.fp)
                 def writeMetadata(self, fp):
                     grp = fp.create_group("Metadata")
                     grp.create_dataset('variables', data=self.tableDim, dtype=self.dtype)
                     for i in range(len(self.metadataList)):
                         if not hasattr(self.dataOut, self.metadataList[i]):
                             log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
                             continue
                         value = getattr(self.dataOut, self.metadataList[i])
                         grp.create_dataset(self.metadataList[i], data=value)
                     return
                 def writeData(self, fp):
                     grp = fp.create_group("Data")
                     dtsets = []
                     data = []
                     for dsInfo in self.dsList:
                         if dsInfo['nDim'] == 0:
                             ds = grp.create_dataset(
                                 dsInfo['variable'],
                                 (self.blocksPerFile, ),
                                 chunks=True,
                                 dtype=numpy.float64)
                             dtsets.append(ds)
                             data.append((dsInfo['variable'], -1))
                         else:
                             sgrp = grp.create_group(dsInfo['variable'])
                             for i in range(dsInfo['dsNumber']):
                                 ds = sgrp.create_dataset(
                                     'table{:02d}'.format(i),
                                     (self.blocksPerFile, ) + dsInfo['shape'][1:],
                                     chunks=True)
                                 dtsets.append(ds)
                                 data.append((dsInfo['variable'], i))
                     fp.flush()
                     log.log('Creating file: {}'.format(fp.filename), self.name)
                     self.ds = dtsets
                     self.data = data
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
                         self.closeFile()
                         self.setNextFile()
                     for i, ds in enumerate(self.ds):
                         attr, ch = self.data[i]
                         if ch == -1:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)
                         else:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
                     self.fp.flush()
                     self.blockIndex += 1
                     log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
                     return
                 def closeFile(self):
                     if self.blockIndex != self.blocksPerFile:
                         for ds in self.ds:
                             ds.resize(self.blockIndex, axis=0)
                     self.fp.flush()
                     self.fp.close()
                 def close(self):
                     self.closeFile()

schainpy/model/proc/jroproc_base.py 0 -3

             '''
             Updated for multiprocessing
             Author : Sergio Cortez
             Jan 2018
             Abstract:
                 Base class for processing units and operations. A decorator provides multiprocessing features and interconnect the processes created.
                 The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
                 The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
             Based on:
                 $Author: murco $
                 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
             '''
             import os
             import sys
             import inspect
             import zmq
             import time
             import pickle
             import traceback
             try:
                 from queue import Queue
             except:
                 from Queue import Queue
             from threading import Thread
             from multiprocessing import Process
             from schainpy.utils import log
             class ProcessingUnit(object):
                 """
                 Update - Jan 2018 - MULTIPROCESSING
                 All the "call" methods present in the previous base were removed.
                 The majority of operations are independant processes, thus
                 the decorator is in charge of communicate the operation processes
                 with the proccessing unit via IPC.
                 The constructor does not receive any argument. The remaining methods
                 are related with the operations to execute.
                 """
                 proc_type = 'processing'
                 __attrs__ = []
                 def __init__(self):
                     self.dataIn = None
                     self.dataOut = None
                     self.isConfig = False
                     self.operations = []
                     self.plots = []
                 def getAllowedArgs(self):
                     if hasattr(self, '__attrs__'):
                         return self.__attrs__
                     else:
                         return inspect.getargspec(self.run).args
                 def addOperation(self, conf, operation):
                     """
                     This method is used in the controller, and update the dictionary containing the operations to execute. The dict
                     posses the id of the operation process (IPC purposes)
                         Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
                         identificador asociado a este objeto.
                         Input:
                             object    :    objeto de la clase "Operation"
                         Return:
                             objId    :    identificador del objeto, necesario para comunicar con master(procUnit)
                     """
                     self.operations.append(
                         (operation, conf.type, conf.id, conf.getKwargs()))
                     if 'plot' in self.name.lower():
                         self.plots.append(operation.CODE)
                 def getOperationObj(self, objId):
                     if objId not in list(self.operations.keys()):
                         return None
                     return self.operations[objId]
                 def operation(self, **kwargs):
                     """
                     Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
                     atributos del objeto dataOut
                     Input:
                         **kwargs    :    Diccionario de argumentos de la funcion a ejecutar
                     """
                     raise NotImplementedError
                 def setup(self):
                     raise NotImplementedError
                 def run(self):
                     raise NotImplementedError
                 def close(self):
                     return
             class Operation(object):
                 """
                 Update - Jan 2018 - MULTIPROCESSING
                 Most of the methods remained the same. The decorator parse the arguments and executed the run() method for each process.
                 The constructor doe snot receive any argument, neither the baseclass.
                     Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
                     y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
                     acumulacion dentro de esta clase
                     Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
                 """
                 proc_type = 'operation'
                 __attrs__ = []
                 def __init__(self):
                     self.id = None
                     self.isConfig = False
                     if not hasattr(self, 'name'):
                         self.name = self.__class__.__name__
                 def getAllowedArgs(self):
                     if hasattr(self, '__attrs__'):
                         return self.__attrs__
                     else:
                         return inspect.getargspec(self.run).args
                 def setup(self):
                     self.isConfig = True
                     raise NotImplementedError
                 def run(self, dataIn, **kwargs):
                     """
                     Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
                     atributos del objeto dataIn.
                     Input:
                         dataIn    :    objeto del tipo JROData
                     Return:
                         None
                     Affected:
                         __buffer    :    buffer de recepcion de datos.
                     """
                     if not self.isConfig:
                         self.setup(**kwargs)
                     raise NotImplementedError
                 def close(self):
                     return
             class InputQueue(Thread):
                 '''
                 Class to hold input data for Proccessing Units and external Operations,
                 '''
                 def __init__(self, project_id, inputId, lock=None):
                     Thread.__init__(self)
                     self.queue = Queue()
                     self.project_id = project_id
                     self.inputId = inputId
                     self.lock = lock
                     self.islocked = False
                     self.size = 0
                 def run(self):
                     c = zmq.Context()
                     self.receiver = c.socket(zmq.SUB)
                     self.receiver.connect(
                         'ipc:///tmp/schain/{}_pub'.format(self.project_id))
                     self.receiver.setsockopt(zmq.SUBSCRIBE, self.inputId.encode())
                     while True:
                         obj = self.receiver.recv_multipart()[1]
                         self.size += sys.getsizeof(obj)
                         self.queue.put(obj)
                 def get(self):
                     if not self.islocked and self.size/1000000 > 512:
                         self.lock.n.value += 1
                         self.islocked = True
                         self.lock.clear()
                     elif self.islocked and self.size/1000000 <= 512:
                         self.islocked = False
                         self.lock.n.value -= 1
                         if self.lock.n.value == 0:
                             self.lock.set()
                     obj = self.queue.get()
                     self.size -= sys.getsizeof(obj)
                     return pickle.loads(obj)
             def MPDecorator(BaseClass):
                 """
                 Multiprocessing class decorator
                 This function add multiprocessing features to a BaseClass. Also, it handle
                 the communication beetween processes (readers, procUnits and operations).
                 """
                 class MPClass(BaseClass, Process):
                     def __init__(self, *args, **kwargs):
                         super(MPClass, self).__init__()
                         Process.__init__(self)
                         self.operationKwargs = {}
                         self.args = args
                         self.kwargs = kwargs
                         self.sender = None
                         self.receiver = None
                         self.i = 0
                         self.t = time.time()
                         self.name = BaseClass.__name__
                         self.__doc__ = BaseClass.__doc__
                         if 'plot' in self.name.lower() and not self.name.endswith('_'):
                             self.name = '{}{}'.format(self.CODE.upper(), 'Plot')
                         self.start_time = time.time()
                         self.id = args[0]
                         self.inputId = args[1]
                         self.project_id = args[2]
                         self.err_queue = args[3]
                         self.lock = args[4]
                         self.typeProc = args[5]
                         self.err_queue.put('#_start_#')
                         if self.inputId is not None:
                             self.queue = InputQueue(self.project_id, self.inputId, self.lock)
                     def subscribe(self):
                         '''
                         Start the zmq socket receiver and subcribe to input ID.
                         '''
                         self.queue.start()
                     def listen(self):
                         '''
                         This function waits for objects
                         '''
                         return self.queue.get()
                     def set_publisher(self):
                         '''
                         This function create a zmq socket for publishing objects.
                         '''
                         time.sleep(0.5)
                         c = zmq.Context()
                         self.sender = c.socket(zmq.PUB)
                         self.sender.connect(
                             'ipc:///tmp/schain/{}_sub'.format(self.project_id))
                     def publish(self, data, id):
                         '''
                         This function publish an object, to an specific topic.
                         It blocks publishing when receiver queue is full to avoid data loss
                         '''
                         if self.inputId is None:
                             self.lock.wait()
                         self.sender.send_multipart([str(id).encode(), pickle.dumps(data)])
                     def runReader(self):
                         '''
                         Run fuction for read units
                         '''
                         while True:
                             try:
                                 BaseClass.run(self, **self.kwargs)
                             except:
                                 err = traceback.format_exc()
                                 if 'No more files' in err:
                                     log.warning('No more files to read', self.name)
                                 else:
                                     self.err_queue.put('{}|{}'.format(self.name, err))
                                 self.dataOut.error = True
                             for op, optype, opId, kwargs in self.operations:
                                 if optype == 'self' and not self.dataOut.flagNoData:
                                     op(**kwargs)
                                 elif optype == 'other' and not self.dataOut.flagNoData:
                                     self.dataOut = op.run(self.dataOut, **self.kwargs)
                                 elif optype == 'external':
                                     self.publish(self.dataOut, opId)
                             if self.dataOut.flagNoData and not self.dataOut.error:
                                 continue
                             self.publish(self.dataOut, self.id)
                             if self.dataOut.error:
                                 break
                         time.sleep(0.5)
                     def runProc(self):
                         '''
                         Run function for proccessing units
                         '''
                         while True:
                             self.dataIn = self.listen()
                             if self.dataIn.flagNoData and self.dataIn.error is None:
                                 continue
                             elif not self.dataIn.error:
                                 try:
                                     BaseClass.run(self, **self.kwargs)
                                 except:
                                     self.err_queue.put('{}|{}'.format(self.name, traceback.format_exc()))
                                     self.dataOut.error = True
                             elif self.dataIn.error:
                                 self.dataOut.error = self.dataIn.error
                                 self.dataOut.flagNoData = True
                             for op, optype, opId, kwargs in self.operations:
                                 if optype == 'self' and not self.dataOut.flagNoData:
                                     op(**kwargs)
                                 elif optype == 'other' and not self.dataOut.flagNoData:
                                     self.dataOut = op.run(self.dataOut, **kwargs)
                                 elif optype == 'external' and not self.dataOut.flagNoData:
                                     self.publish(self.dataOut, opId)
                             self.publish(self.dataOut, self.id)
                             for op, optype, opId, kwargs in self.operations:
                                 if optype == 'external' and self.dataOut.error:
                                     self.publish(self.dataOut, opId)
                             if self.dataOut.error:
                                 break
                         time.sleep(0.5)
                     def runOp(self):
                         '''
                         Run function for external operations (this operations just receive data
                         ex: plots, writers, publishers)
                         '''
                         while True:
                             dataOut = self.listen()
                             if not dataOut.error:
                                 try:
                                     BaseClass.run(self, dataOut, **self.kwargs)
                                 except:
                                     self.err_queue.put('{}|{}'.format(self.name, traceback.format_exc()))
                                     dataOut.error = True
                             else:
                                 break
                     def run(self):
                         if self.typeProc is "ProcUnit":
                             if self.inputId is not None:
                                 self.subscribe()
                             self.set_publisher()
                             if 'Reader' not in BaseClass.__name__:
                                 self.runProc()
                             else:
                                 self.runReader()
                         elif self.typeProc is "Operation":
                             self.subscribe()
                             self.runOp()
                         else:
                             raise ValueError("Unknown type")
                         self.close()
                     def close(self):
                         BaseClass.close(self)
                         self.err_queue.put('#_end_#')
                         if self.sender:
                             self.sender.close()
                         if self.receiver:
                             self.receiver.close()
                         log.success('Done...(Time:{:4.2f} secs)'.format(time.time()-self.start_time), self.name)
                 return MPClass

schainpy/model/proc/jroproc_parameters.py +1 -1

             import numpy
             import math
             from scipy import optimize, interpolate, signal, stats, ndimage
             import scipy
             import re
             import datetime
             import copy
             import sys
             import importlib
             import itertools
             from multiprocessing import Pool, TimeoutError
             from multiprocessing.pool import ThreadPool
             import time
             from scipy.optimize import fmin_l_bfgs_b #optimize with bounds on state papameters
             from .jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jrodata import Parameters, hildebrand_sekhon
             from scipy import asarray as ar,exp
             from scipy.optimize import curve_fit
             from schainpy.utils import log
             import warnings
             from numpy import NaN
             from scipy.optimize.optimize import OptimizeWarning
             warnings.filterwarnings('ignore')
             SPEED_OF_LIGHT = 299792458
             '''solving pickling issue'''
             def _pickle_method(method):
                 func_name = method.__func__.__name__
                 obj = method.__self__
                 cls = method.__self__.__class__
                 return _unpickle_method, (func_name, obj, cls)
             def _unpickle_method(func_name, obj, cls):
                 for cls in cls.mro():
                     try:
                         func = cls.__dict__[func_name]
                     except KeyError:
                         pass
                     else:
                         break
                 return func.__get__(obj, cls)
             @MPDecorator
             class ParametersProc(ProcessingUnit):
                 METHODS = {}
                 nSeconds = None
                 def __init__(self):
                     ProcessingUnit.__init__(self)
             #         self.objectDict = {}
                     self.buffer = None
                     self.firstdatatime = None
                     self.profIndex = 0
                     self.dataOut = Parameters()
                     self.setupReq = False #Agregar a todas las unidades de proc
                 def __updateObjFromInput(self):
                     self.dataOut.inputUnit = self.dataIn.type
                     self.dataOut.timeZone = self.dataIn.timeZone
                     self.dataOut.dstFlag = self.dataIn.dstFlag
                     self.dataOut.errorCount = self.dataIn.errorCount
                     self.dataOut.useLocalTime = self.dataIn.useLocalTime
                     self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
                     self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
                     self.dataOut.channelList = self.dataIn.channelList
                     self.dataOut.heightList = self.dataIn.heightList
                     self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
             #         self.dataOut.nHeights = self.dataIn.nHeights
             #         self.dataOut.nChannels = self.dataIn.nChannels
                     self.dataOut.nBaud = self.dataIn.nBaud
                     self.dataOut.nCode = self.dataIn.nCode
                     self.dataOut.code = self.dataIn.code
             #        self.dataOut.nProfiles = self.dataOut.nFFTPoints
                     self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
             #         self.dataOut.utctime = self.firstdatatime
                     self.dataOut.utctime = self.dataIn.utctime
                     self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
                     self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
                     self.dataOut.nCohInt = self.dataIn.nCohInt
             #        self.dataOut.nIncohInt = 1
                     self.dataOut.ippSeconds = self.dataIn.ippSeconds
             #        self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
                     self.dataOut.timeInterval1 = self.dataIn.timeInterval
                     self.dataOut.heightList = self.dataIn.getHeiRange()
                     self.dataOut.frequency = self.dataIn.frequency
                     # self.dataOut.noise = self.dataIn.noise
                 def run(self):
                     #----------------------    Voltage Data    ---------------------------
                     if self.dataIn.type == "Voltage":
                         self.__updateObjFromInput()
                         self.dataOut.data_pre = self.dataIn.data.copy()
                         self.dataOut.flagNoData = False
                         self.dataOut.utctimeInit = self.dataIn.utctime
                         self.dataOut.paramInterval = self.dataIn.nProfiles*self.dataIn.nCohInt*self.dataIn.ippSeconds
                         return
                     #----------------------    Spectra Data    ---------------------------
                     if self.dataIn.type == "Spectra":
                         self.dataOut.data_pre = (self.dataIn.data_spc, self.dataIn.data_cspc)
                         self.dataOut.data_spc = self.dataIn.data_spc
                         self.dataOut.data_cspc = self.dataIn.data_cspc
                         self.dataOut.nProfiles = self.dataIn.nProfiles
                         self.dataOut.nIncohInt = self.dataIn.nIncohInt
                         self.dataOut.nFFTPoints = self.dataIn.nFFTPoints
                         self.dataOut.ippFactor = self.dataIn.ippFactor
                         self.dataOut.abscissaList = self.dataIn.getVelRange(1)
                         self.dataOut.spc_noise = self.dataIn.getNoise()
                         self.dataOut.spc_range = (self.dataIn.getFreqRange(1) , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1))
                         # self.dataOut.normFactor = self.dataIn.normFactor
                         self.dataOut.pairsList = self.dataIn.pairsList
                         self.dataOut.groupList = self.dataIn.pairsList
                         self.dataOut.flagNoData = False
                         if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels
                             self.dataOut.ChanDist = self.dataIn.ChanDist
                         else: self.dataOut.ChanDist = None
                         #if hasattr(self.dataIn, 'VelRange'): #Velocities range
                         #    self.dataOut.VelRange = self.dataIn.VelRange
                         #else: self.dataOut.VelRange = None
                         if hasattr(self.dataIn, 'RadarConst'): #Radar Constant
                             self.dataOut.RadarConst = self.dataIn.RadarConst
                         if hasattr(self.dataIn, 'NPW'): #NPW
                             self.dataOut.NPW = self.dataIn.NPW
                         if hasattr(self.dataIn, 'COFA'): #COFA
                             self.dataOut.COFA = self.dataIn.COFA
                     #----------------------    Correlation Data    ---------------------------
                     if self.dataIn.type == "Correlation":
                         acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.dataIn.splitFunctions()
                         self.dataOut.data_pre = (self.dataIn.data_cf[acf_ind,:], self.dataIn.data_cf[ccf_ind,:,:])
                         self.dataOut.normFactor = (self.dataIn.normFactor[acf_ind,:], self.dataIn.normFactor[ccf_ind,:])
                         self.dataOut.groupList = (acf_pairs, ccf_pairs)
                         self.dataOut.abscissaList = self.dataIn.lagRange
                         self.dataOut.noise = self.dataIn.noise
                         self.dataOut.data_SNR = self.dataIn.SNR
                         self.dataOut.flagNoData = False
                         self.dataOut.nAvg = self.dataIn.nAvg
                     #----------------------    Parameters Data    ---------------------------
                     if self.dataIn.type == "Parameters":
                         self.dataOut.copy(self.dataIn)
                         self.dataOut.flagNoData = False
                         return True
                     self.__updateObjFromInput()
                     self.dataOut.utctimeInit = self.dataIn.utctime
                     self.dataOut.paramInterval = self.dataIn.timeInterval
                     return
             def target(tups):
                 obj, args = tups
                 return obj.FitGau(args)
             class SpectralFilters(Operation):
                 '''This class allows the Rainfall / Wind Selection for CLAIRE RADAR
                     LimitR :    It is the limit in m/s of Rainfall
                     LimitW :    It is the limit in m/s for Winds
                     Input:
                     self.dataOut.data_pre :        SPC and CSPC
                     self.dataOut.spc_range :       To select wind and rainfall velocities
                     Affected:
                     self.dataOut.data_pre :        It is used for the new SPC and CSPC ranges of wind
                     self.dataOut.spcparam_range :  Used in SpcParamPlot
                     self.dataOut.SPCparam :        Used in PrecipitationProc
                 '''
                 def __init__(self):
                     Operation.__init__(self)
                     self.i=0
                 def run(self, dataOut, PositiveLimit=1.5, NegativeLimit=2.5):
                     #Limite de vientos
                     LimitR = PositiveLimit
                     LimitN = NegativeLimit
                     self.spc = dataOut.data_pre[0].copy()
                     self.cspc = dataOut.data_pre[1].copy()
                     self.Num_Hei = self.spc.shape[2]
                     self.Num_Bin = self.spc.shape[1]
                     self.Num_Chn = self.spc.shape[0]
                     VelRange = dataOut.spc_range[2]
                     TimeRange = dataOut.spc_range[1]
                     FrecRange = dataOut.spc_range[0]
                     Vmax= 2*numpy.max(dataOut.spc_range[2])
                     Tmax= 2*numpy.max(dataOut.spc_range[1])
                     Fmax= 2*numpy.max(dataOut.spc_range[0])
                     Breaker1R=VelRange[numpy.abs(VelRange-(-LimitN)).argmin()]
                     Breaker1R=numpy.where(VelRange == Breaker1R)
                     Delta = self.Num_Bin/2 - Breaker1R[0]
                     '''Reacomodando SPCrange'''
                     VelRange=numpy.roll(VelRange,-(int(self.Num_Bin/2)) ,axis=0)
                     VelRange[-(int(self.Num_Bin/2)):]+= Vmax
                     FrecRange=numpy.roll(FrecRange,-(int(self.Num_Bin/2)),axis=0)
                     FrecRange[-(int(self.Num_Bin/2)):]+= Fmax
                     TimeRange=numpy.roll(TimeRange,-(int(self.Num_Bin/2)),axis=0)
                     TimeRange[-(int(self.Num_Bin/2)):]+= Tmax
                     ''' ------------------ '''
                     Breaker2R=VelRange[numpy.abs(VelRange-(LimitR)).argmin()]
                     Breaker2R=numpy.where(VelRange == Breaker2R)
                     SPCroll = numpy.roll(self.spc,-(int(self.Num_Bin/2)) ,axis=1)
                     SPCcut = SPCroll.copy()
                     for i in range(self.Num_Chn):
                         SPCcut[i,0:int(Breaker2R[0]),:] = dataOut.noise[i]
                         SPCcut[i,-int(Delta):,:] = dataOut.noise[i]
                         SPCcut[i]=SPCcut[i]- dataOut.noise[i]
                         SPCcut[ numpy.where( SPCcut<0 ) ] = 1e-20
                         SPCroll[i]=SPCroll[i]-dataOut.noise[i]
                         SPCroll[ numpy.where( SPCroll<0 ) ] = 1e-20
                     SPC_ch1 = SPCroll
                     SPC_ch2 = SPCcut
                     SPCparam = (SPC_ch1, SPC_ch2, self.spc)
                     dataOut.SPCparam = numpy.asarray(SPCparam)
                     dataOut.spcparam_range=numpy.zeros([self.Num_Chn,self.Num_Bin+1])
                     dataOut.spcparam_range[2]=VelRange
                     dataOut.spcparam_range[1]=TimeRange
                     dataOut.spcparam_range[0]=FrecRange
                     return dataOut
             class GaussianFit(Operation):
                 '''
                     Function that fit of one and two generalized gaussians (gg) based
                     on the PSD shape across an "power band" identified from a cumsum of
                     the measured spectrum - noise.
                     Input:
                         self.dataOut.data_pre    :    SelfSpectra
                     Output:
                         self.dataOut.SPCparam :    SPC_ch1, SPC_ch2
                 '''
                 def __init__(self):
                     Operation.__init__(self)
                     self.i=0
                 def run(self, dataOut, num_intg=7, pnoise=1., SNRlimit=-9): #num_intg: Incoherent integrations, pnoise: Noise, vel_arr: range of velocities, similar to the ftt points
                     """This routine will find a couple of generalized Gaussians to a power spectrum
                     input: spc
                     output:
                         Amplitude0,shift0,width0,p0,Amplitude1,shift1,width1,p1,noise
                     """
                     self.spc = dataOut.data_pre[0].copy()
                     self.Num_Hei = self.spc.shape[2]
                     self.Num_Bin = self.spc.shape[1]
                     self.Num_Chn = self.spc.shape[0]
                     Vrange = dataOut.abscissaList
                     GauSPC = numpy.empty([self.Num_Chn,self.Num_Bin,self.Num_Hei])
                     SPC_ch1 = numpy.empty([self.Num_Bin,self.Num_Hei])
                     SPC_ch2 = numpy.empty([self.Num_Bin,self.Num_Hei])
                     SPC_ch1[:] = numpy.NaN
                     SPC_ch2[:] = numpy.NaN
                     start_time = time.time()
                     noise_ = dataOut.spc_noise[0].copy()
                     pool = Pool(processes=self.Num_Chn)
                     args = [(Vrange, Ch, pnoise, noise_, num_intg, SNRlimit) for Ch in range(self.Num_Chn)]
                     objs = [self for __ in range(self.Num_Chn)]
                     attrs = list(zip(objs, args))
                     gauSPC = pool.map(target, attrs)
                     dataOut.SPCparam = numpy.asarray(SPCparam)
                     ''' Parameters:
 . Amplitude
 . Shift
 . Width
 . Power
                            '''
                 def FitGau(self, X):
                     Vrange, ch, pnoise, noise_, num_intg, SNRlimit = X
                     SPCparam = []
                     SPC_ch1 = numpy.empty([self.Num_Bin,self.Num_Hei])
                     SPC_ch2 = numpy.empty([self.Num_Bin,self.Num_Hei])
                     SPC_ch1[:] = 0#numpy.NaN
                     SPC_ch2[:] = 0#numpy.NaN
                     for ht in range(self.Num_Hei):
                         spc =  numpy.asarray(self.spc)[ch,:,ht]
                         #############################################
                         # normalizing spc and noise
                         # This part differs from gg1
                         spc_norm_max = max(spc)
                         #spc = spc / spc_norm_max
                         pnoise = pnoise #/ spc_norm_max
                         #############################################
                         fatspectra=1.0
                         wnoise = noise_ #/ spc_norm_max
                             #wnoise,stdv,i_max,index =enoise(spc,num_intg) #noise estimate using Hildebrand Sekhon, only wnoise is used
                             #if wnoise>1.1*pnoise: # to be tested later
                             #    wnoise=pnoise
                         noisebl=wnoise*0.9;
                         noisebh=wnoise*1.1
                         spc=spc-wnoise
                         minx=numpy.argmin(spc)
                         #spcs=spc.copy()
                         spcs=numpy.roll(spc,-minx)
                         cum=numpy.cumsum(spcs)
                         tot_noise=wnoise * self.Num_Bin  #64;
                         snr = sum(spcs)/tot_noise
                         snrdB=10.*numpy.log10(snr)
                         if snrdB < SNRlimit :
                             snr = numpy.NaN
                             SPC_ch1[:,ht] = 0#numpy.NaN
                             SPC_ch1[:,ht] = 0#numpy.NaN
                             SPCparam = (SPC_ch1,SPC_ch2)
                             continue
                         #if snrdB<-18 or numpy.isnan(snrdB) or num_intg<4:
                         #    return [None,]*4,[None,]*4,None,snrdB,None,None,[None,]*5,[None,]*9,None
                         cummax=max(cum);
                         epsi=0.08*fatspectra # cumsum to narrow down the energy region
                         cumlo=cummax*epsi;
                         cumhi=cummax*(1-epsi)
                         powerindex=numpy.array(numpy.where(numpy.logical_and(cum>cumlo, cum<cumhi))[0])
                         if len(powerindex) < 1:# case for powerindex 0
                             continue
                         powerlo=powerindex[0]
                         powerhi=powerindex[-1]
                         powerwidth=powerhi-powerlo
                         firstpeak=powerlo+powerwidth/10.# first gaussian energy location
                         secondpeak=powerhi-powerwidth/10.#second gaussian energy location
                         midpeak=(firstpeak+secondpeak)/2.
                         firstamp=spcs[int(firstpeak)]
                         secondamp=spcs[int(secondpeak)]
                         midamp=spcs[int(midpeak)]
                         x=numpy.arange( self.Num_Bin )
                         y_data=spc+wnoise
                         '''    single Gaussian    '''
                         shift0=numpy.mod(midpeak+minx, self.Num_Bin )
                         width0=powerwidth/4.#Initialization entire power of spectrum divided by 4
                         power0=2.
                         amplitude0=midamp
                         state0=[shift0,width0,amplitude0,power0,wnoise]
                         bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth),(0,None),(0.5,3.),(noisebl,noisebh))
                         lsq1=fmin_l_bfgs_b(self.misfit1,state0,args=(y_data,x,num_intg),bounds=bnds,approx_grad=True)
                         chiSq1=lsq1[1];
                         if fatspectra<1.0 and powerwidth<4:
                                 choice=0
                                 Amplitude0=lsq1[0][2]
                                 shift0=lsq1[0][0]
                                 width0=lsq1[0][1]
                                 p0=lsq1[0][3]
                                 Amplitude1=0.
                                 shift1=0.
                                 width1=0.
                                 p1=0.
                                 noise=lsq1[0][4]
                                 #return (numpy.array([shift0,width0,Amplitude0,p0]),
                                 #        numpy.array([shift1,width1,Amplitude1,p1]),noise,snrdB,chiSq1,6.,sigmas1,[None,]*9,choice)
                         '''    two gaussians    '''
                         #shift0=numpy.mod(firstpeak+minx,64); shift1=numpy.mod(secondpeak+minx,64)
                         shift0=numpy.mod(firstpeak+minx, self.Num_Bin );
                         shift1=numpy.mod(secondpeak+minx, self.Num_Bin )
                         width0=powerwidth/6.;
                         width1=width0
                         power0=2.;
                         power1=power0
                         amplitude0=firstamp;
                         amplitude1=secondamp
                         state0=[shift0,width0,amplitude0,power0,shift1,width1,amplitude1,power1,wnoise]
                         #bnds=((0,63),(1,powerwidth/2.),(0,None),(0.5,3.),(0,63),(1,powerwidth/2.),(0,None),(0.5,3.),(noisebl,noisebh))
                         bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth/2.),(0,None),(0.5,3.),( 0,(self.Num_Bin-1)),(1,powerwidth/2.),(0,None),(0.5,3.),(noisebl,noisebh))
                         #bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth/2.),(0,None),(0.5,3.),( 0,(self.Num_Bin-1)),(1,powerwidth/2.),(0,None),(0.5,3.),(0.1,0.5))
                         lsq2 = fmin_l_bfgs_b( self.misfit2 , state0 , args=(y_data,x,num_intg) , bounds=bnds , approx_grad=True )
                         chiSq2=lsq2[1];
                         oneG=(chiSq1<5 and chiSq1/chiSq2<2.0) and (abs(lsq2[0][0]-lsq2[0][4])<(lsq2[0][1]+lsq2[0][5])/3. or abs(lsq2[0][0]-lsq2[0][4])<10)
                         if snrdB>-12: # when SNR is strong pick the peak with least shift (LOS velocity) error
                             if oneG:
                                 choice=0
                             else:
                                 w1=lsq2[0][1]; w2=lsq2[0][5]
                                 a1=lsq2[0][2]; a2=lsq2[0][6]
                                 p1=lsq2[0][3]; p2=lsq2[0][7]
                                 s1=(2**(1+1./p1))*scipy.special.gamma(1./p1)/p1;
                                 s2=(2**(1+1./p2))*scipy.special.gamma(1./p2)/p2;
                                 gp1=a1*w1*s1; gp2=a2*w2*s2 # power content of each ggaussian with proper p scaling
                                 if gp1>gp2:
                                     if a1>0.7*a2:
                                         choice=1
                                     else:
                                         choice=2
                                 elif gp2>gp1:
                                     if a2>0.7*a1:
                                         choice=2
                                     else:
                                         choice=1
                                 else:
                                     choice=numpy.argmax([a1,a2])+1
                                     #else:
                                     #choice=argmin([std2a,std2b])+1
                         else: # with low SNR go to the most energetic peak
                             choice=numpy.argmax([lsq1[0][2]*lsq1[0][1],lsq2[0][2]*lsq2[0][1],lsq2[0][6]*lsq2[0][5]])
                         shift0=lsq2[0][0];
                         vel0=Vrange[0] + shift0*(Vrange[1]-Vrange[0])
                         shift1=lsq2[0][4];
                         vel1=Vrange[0] + shift1*(Vrange[1]-Vrange[0])
                         max_vel = 1.0
                         #first peak will be 0, second peak will be 1
                         if vel0 > -1.0 and vel0 < max_vel : #first peak is in the correct range
                             shift0=lsq2[0][0]
                             width0=lsq2[0][1]
                             Amplitude0=lsq2[0][2]
                             p0=lsq2[0][3]
                             shift1=lsq2[0][4]
                             width1=lsq2[0][5]
                             Amplitude1=lsq2[0][6]
                             p1=lsq2[0][7]
                             noise=lsq2[0][8]
                         else:
                             shift1=lsq2[0][0]
                             width1=lsq2[0][1]
                             Amplitude1=lsq2[0][2]
                             p1=lsq2[0][3]
                             shift0=lsq2[0][4]
                             width0=lsq2[0][5]
                             Amplitude0=lsq2[0][6]
                             p0=lsq2[0][7]
                             noise=lsq2[0][8]
                         if Amplitude0<0.05: # in case the peak is noise
                             shift0,width0,Amplitude0,p0 = [0,0,0,0]#4*[numpy.NaN]
                         if Amplitude1<0.05:
                             shift1,width1,Amplitude1,p1 = [0,0,0,0]#4*[numpy.NaN]
                         SPC_ch1[:,ht] = noise + Amplitude0*numpy.exp(-0.5*(abs(x-shift0))/width0)**p0
                         SPC_ch2[:,ht] = noise + Amplitude1*numpy.exp(-0.5*(abs(x-shift1))/width1)**p1
                         SPCparam = (SPC_ch1,SPC_ch2)
                     return GauSPC
                 def y_model1(self,x,state):
                     shift0,width0,amplitude0,power0,noise=state
                     model0=amplitude0*numpy.exp(-0.5*abs((x-shift0)/width0)**power0)
                     model0u=amplitude0*numpy.exp(-0.5*abs((x-shift0- self.Num_Bin )/width0)**power0)
                     model0d=amplitude0*numpy.exp(-0.5*abs((x-shift0+ self.Num_Bin )/width0)**power0)
                     return model0+model0u+model0d+noise
                 def y_model2(self,x,state): #Equation for two generalized Gaussians with Nyquist
                     shift0,width0,amplitude0,power0,shift1,width1,amplitude1,power1,noise=state
                     model0=amplitude0*numpy.exp(-0.5*abs((x-shift0)/width0)**power0)
                     model0u=amplitude0*numpy.exp(-0.5*abs((x-shift0- self.Num_Bin )/width0)**power0)
                     model0d=amplitude0*numpy.exp(-0.5*abs((x-shift0+ self.Num_Bin )/width0)**power0)
                     model1=amplitude1*numpy.exp(-0.5*abs((x-shift1)/width1)**power1)
                     model1u=amplitude1*numpy.exp(-0.5*abs((x-shift1- self.Num_Bin )/width1)**power1)
                     model1d=amplitude1*numpy.exp(-0.5*abs((x-shift1+ self.Num_Bin )/width1)**power1)
                     return model0+model0u+model0d+model1+model1u+model1d+noise
                 def misfit1(self,state,y_data,x,num_intg): # This function compares how close real data is with the model data, the close it is, the better it is.
                     return num_intg*sum((numpy.log(y_data)-numpy.log(self.y_model1(x,state)))**2)#/(64-5.) # /(64-5.) can be commented
                 def misfit2(self,state,y_data,x,num_intg):
                     return num_intg*sum((numpy.log(y_data)-numpy.log(self.y_model2(x,state)))**2)#/(64-9.)
             class PrecipitationProc(Operation):
                 '''
                      Operator that estimates Reflectivity factor (Z), and estimates rainfall Rate (R)
                      Input:
                         self.dataOut.data_pre    :    SelfSpectra
                      Output:
                         self.dataOut.data_output :    Reflectivity factor, rainfall Rate
                      Parameters affected:
                 '''
                 def __init__(self):
                     Operation.__init__(self)
                     self.i=0
                 def gaus(self,xSamples,Amp,Mu,Sigma):
                     return ( Amp / ((2*numpy.pi)**0.5 * Sigma) ) * numpy.exp( -( xSamples - Mu )**2 / ( 2 * (Sigma**2) ))
                 def Moments(self, ySamples, xSamples):
                     Pot = numpy.nansum( ySamples )                              # Potencia, momento 0
                     yNorm = ySamples / Pot
                     Vr = numpy.nansum( yNorm * xSamples )                       # Velocidad radial, mu, corrimiento doppler, primer momento
                     Sigma2 = abs(numpy.nansum( yNorm * ( xSamples - Vr )**2 ))  # Segundo Momento
                     Desv = Sigma2**0.5                                          # Desv. Estandar, Ancho espectral
                     return numpy.array([Pot, Vr, Desv])
                 def run(self, dataOut, radar=None, Pt=5000, Gt=295.1209, Gr=70.7945, Lambda=0.6741, aL=2.5118,
                         tauW=4e-06, ThetaT=0.1656317, ThetaR=0.36774087, Km = 0.93, Altitude=3350):
                     Velrange = dataOut.spcparam_range[2]
                     FrecRange = dataOut.spcparam_range[0]
                     dV= Velrange[1]-Velrange[0]
                     dF= FrecRange[1]-FrecRange[0]
                     if radar == "MIRA35C" :
                         self.spc = dataOut.data_pre[0].copy()
                         self.Num_Hei = self.spc.shape[2]
                         self.Num_Bin = self.spc.shape[1]
                         self.Num_Chn = self.spc.shape[0]
                         Ze = self.dBZeMODE2(dataOut)
                     else:
                         self.spc = dataOut.SPCparam[1].copy() #dataOut.data_pre[0].copy() #
                         """NOTA SE DEBE REMOVER EL RANGO DEL PULSO TX"""
                         self.spc[:,:,0:7]= numpy.NaN
                         """##########################################"""
                         self.Num_Hei = self.spc.shape[2]
                         self.Num_Bin = self.spc.shape[1]
                         self.Num_Chn = self.spc.shape[0]
                         ''' Se obtiene la constante del RADAR '''
                         self.Pt = Pt
                         self.Gt = Gt
                         self.Gr = Gr
                         self.Lambda = Lambda
                         self.aL = aL
                         self.tauW = tauW
                         self.ThetaT = ThetaT
                         self.ThetaR = ThetaR
                         Numerator = ( (4*numpy.pi)**3 * aL**2 * 16 * numpy.log(2) )
                         Denominator = ( Pt * Gt * Gr * Lambda**2 * SPEED_OF_LIGHT * tauW * numpy.pi * ThetaT * ThetaR)
                         RadarConstant = 10e-26 * Numerator / Denominator #
                         ''' =============================  '''
                         self.spc[0] = (self.spc[0]-dataOut.noise[0])
                         self.spc[1] = (self.spc[1]-dataOut.noise[1])
                         self.spc[2] = (self.spc[2]-dataOut.noise[2])
                         self.spc[ numpy.where(self.spc < 0)] = 0
                         SPCmean = (numpy.mean(self.spc,0) - numpy.mean(dataOut.noise))
                         SPCmean[ numpy.where(SPCmean < 0)] = 0
                         ETAn = numpy.zeros([self.Num_Bin,self.Num_Hei])
                         ETAv = numpy.zeros([self.Num_Bin,self.Num_Hei])
                         ETAd = numpy.zeros([self.Num_Bin,self.Num_Hei])
                         Pr = SPCmean[:,:]
                         VelMeteoro = numpy.mean(SPCmean,axis=0)
                         D_range = numpy.zeros([self.Num_Bin,self.Num_Hei])
                         SIGMA = numpy.zeros([self.Num_Bin,self.Num_Hei])
                         N_dist = numpy.zeros([self.Num_Bin,self.Num_Hei])
                         V_mean = numpy.zeros(self.Num_Hei)
                         del_V = numpy.zeros(self.Num_Hei)
                         Z = numpy.zeros(self.Num_Hei)
                         Ze = numpy.zeros(self.Num_Hei)
                         RR = numpy.zeros(self.Num_Hei)
                         Range = dataOut.heightList*1000.
                         for R in range(self.Num_Hei):
                             h = Range[R] + Altitude #Range from ground to radar pulse altitude
                             del_V[R] = 1 + 3.68 * 10**-5 * h + 1.71 * 10**-9 * h**2    #Density change correction for velocity
                             D_range[:,R] = numpy.log( (9.65 - (Velrange[0:self.Num_Bin] / del_V[R])) / 10.3 ) / -0.6 #Diameter range [m]x10**-3
                             '''NOTA: ETA(n) dn = ETA(f) df
                                 dn = 1     Diferencial de muestreo
                                 df = ETA(n) / ETA(f)
                             '''
                             ETAn[:,R] = RadarConstant * Pr[:,R] * (Range[R] )**2 #Reflectivity (ETA)
                             ETAv[:,R]=ETAn[:,R]/dV
                             ETAd[:,R]=ETAv[:,R]*6.18*exp(-0.6*D_range[:,R])
                             SIGMA[:,R] = Km * (D_range[:,R] * 1e-3 )**6 * numpy.pi**5 / Lambda**4      #Equivalent Section of drops (sigma)
                             N_dist[:,R] = ETAn[:,R] / SIGMA[:,R]
                             DMoments = self.Moments(Pr[:,R], Velrange[0:self.Num_Bin])
                             try:
                                 popt01,pcov = curve_fit(self.gaus, Velrange[0:self.Num_Bin] , Pr[:,R] , p0=DMoments)
                             except:
                                 popt01=numpy.zeros(3)
                                 popt01[1]= DMoments[1]
                             if popt01[1]<0 or popt01[1]>20:
                                 popt01[1]=numpy.NaN
                             V_mean[R]=popt01[1]
                             Z[R] = numpy.nansum( N_dist[:,R] * (D_range[:,R])**6 )#*10**-18
                             RR[R] = 0.0006*numpy.pi * numpy.nansum( D_range[:,R]**3 * N_dist[:,R] * Velrange[0:self.Num_Bin] ) #Rainfall rate
                             Ze[R] = (numpy.nansum( ETAn[:,R]) * Lambda**4) / ( 10**-18*numpy.pi**5 * Km)
                     RR2 = (Z/200)**(1/1.6)
                     dBRR = 10*numpy.log10(RR)
                     dBRR2 = 10*numpy.log10(RR2)
                     dBZe = 10*numpy.log10(Ze)
                     dBZ = 10*numpy.log10(Z)
                     dataOut.data_output = RR[8]
                     dataOut.data_param = numpy.ones([3,self.Num_Hei])
                     dataOut.channelList = [0,1,2]
                     dataOut.data_param[0]=dBZ
                     dataOut.data_param[1]=V_mean
                     dataOut.data_param[2]=RR
                     return dataOut
                 def dBZeMODE2(self, dataOut): #    Processing for MIRA35C
                     NPW = dataOut.NPW
                     COFA = dataOut.COFA
                     SNR = numpy.array([self.spc[0,:,:] / NPW[0]]) #, self.spc[1,:,:] / NPW[1]])
                     RadarConst = dataOut.RadarConst
                     #frequency = 34.85*10**9
                     ETA = numpy.zeros(([self.Num_Chn ,self.Num_Hei]))
                     data_output = numpy.ones([self.Num_Chn , self.Num_Hei])*numpy.NaN
                     ETA = numpy.sum(SNR,1)
                     ETA = numpy.where(ETA is not 0. , ETA, numpy.NaN)
                     Ze = numpy.ones([self.Num_Chn, self.Num_Hei] )
                     for r in range(self.Num_Hei):
                         Ze[0,r] =  ( ETA[0,r] ) * COFA[0,r][0] * RadarConst * ((r/5000.)**2)
                         #Ze[1,r] =  ( ETA[1,r] ) * COFA[1,r][0] * RadarConst * ((r/5000.)**2)
                     return Ze
             #     def GetRadarConstant(self):
             #
             #         """
             #         Constants:
             #
             #         Pt:     Transmission Power               dB        5kW                5000
             #         Gt:     Transmission Gain                dB        24.7 dB            295.1209
             #         Gr:     Reception Gain                   dB        18.5 dB            70.7945
             #         Lambda: Wavelenght                       m         0.6741 m           0.6741
             #         aL:     Attenuation loses                dB        4dB                2.5118
             #         tauW:   Width of transmission pulse      s         4us                4e-6
             #         ThetaT: Transmission antenna bean angle  rad       0.1656317 rad      0.1656317
             #         ThetaR: Reception antenna beam angle     rad       0.36774087 rad     0.36774087
             #
             #         """
             #
             #         Numerator = ( (4*numpy.pi)**3 * aL**2 * 16 * numpy.log(2) )
             #         Denominator = ( Pt * Gt * Gr * Lambda**2 * SPEED_OF_LIGHT * TauW * numpy.pi * ThetaT * TheraR)
             #         RadarConstant =  Numerator / Denominator
             #
             #         return RadarConstant
             class FullSpectralAnalysis(Operation):
                 """
                     Function that implements Full Spectral Analisys technique.
                     Input:
                         self.dataOut.data_pre    :    SelfSpectra and CrossSPectra data
                         self.dataOut.groupList   :    Pairlist of channels
                         self.dataOut.ChanDist    :    Physical distance between receivers
                     Output:
                         self.dataOut.data_output :    Zonal wind, Meridional wind and Vertical wind
                     Parameters affected:    Winds, height range, SNR
                 """
                 def run(self, dataOut, Xi01=None, Xi02=None, Xi12=None, Eta01=None, Eta02=None, Eta12=None, SNRlimit=7):
                     self.indice=int(numpy.random.rand()*1000)
                     spc = dataOut.data_pre[0].copy()
                     cspc = dataOut.data_pre[1]
                     """NOTA SE DEBE REMOVER EL RANGO DEL PULSO TX"""
                     SNRspc = spc.copy()
                     SNRspc[:,:,0:7]= numpy.NaN
                     """##########################################"""
                     nChannel = spc.shape[0]
                     nProfiles = spc.shape[1]
                     nHeights = spc.shape[2]
                     pairsList = dataOut.groupList
                     if dataOut.ChanDist is not None :
                         ChanDist = dataOut.ChanDist
                     else:
                         ChanDist = numpy.array([[Xi01, Eta01],[Xi02,Eta02],[Xi12,Eta12]])
                     FrecRange = dataOut.spc_range[0]
                     ySamples=numpy.ones([nChannel,nProfiles])
                     phase=numpy.ones([nChannel,nProfiles])
                     CSPCSamples=numpy.ones([nChannel,nProfiles],dtype=numpy.complex_)
                     coherence=numpy.ones([nChannel,nProfiles])
                     PhaseSlope=numpy.ones(nChannel)
                     PhaseInter=numpy.ones(nChannel)
                     data_SNR=numpy.zeros([nProfiles])
                     data = dataOut.data_pre
                     noise = dataOut.noise
                     dataOut.data_SNR = (numpy.mean(SNRspc,axis=1)- noise[0]) / noise[0]
                     dataOut.data_SNR[numpy.where( dataOut.data_SNR <0 )] = 1e-20
                     data_output=numpy.ones([spc.shape[0],spc.shape[2]])*numpy.NaN
                     velocityX=[]
                     velocityY=[]
                     velocityV=[]
                     PhaseLine=[]
                     dbSNR = 10*numpy.log10(dataOut.data_SNR)
                     dbSNR = numpy.average(dbSNR,0)
                     for Height in range(nHeights):
                         [Vzon,Vmer,Vver, GaussCenter, PhaseSlope, FitGaussCSPC]= self.WindEstimation(spc, cspc, pairsList, ChanDist, Height, noise, dataOut.spc_range, dbSNR[Height], SNRlimit)
                         PhaseLine = numpy.append(PhaseLine, PhaseSlope)
                         if abs(Vzon)<100. and abs(Vzon)> 0.:
                             velocityX=numpy.append(velocityX, Vzon)#Vmag
                         else:
                             velocityX=numpy.append(velocityX, numpy.NaN)
                         if abs(Vmer)<100. and abs(Vmer) > 0.:
                             velocityY=numpy.append(velocityY, -Vmer)#Vang
                         else:
                             velocityY=numpy.append(velocityY, numpy.NaN)
                         if dbSNR[Height] > SNRlimit:
                             velocityV=numpy.append(velocityV, -Vver)#FirstMoment[Height])
                         else:
                             velocityV=numpy.append(velocityV, numpy.NaN)
                     '''Nota: Cambiar el signo de numpy.array(velocityX) cuando se intente procesar datos de BLTR'''
                     data_output[0] = numpy.array(velocityX)  #self.moving_average(numpy.array(velocityX) , N=1)
                     data_output[1] = numpy.array(velocityY)  #self.moving_average(numpy.array(velocityY) , N=1)
                     data_output[2] = velocityV#FirstMoment
                     xFrec=FrecRange[0:spc.shape[1]]
                     dataOut.data_output=data_output
                     return dataOut
                 def moving_average(self,x, N=2):
                     return numpy.convolve(x, numpy.ones((N,))/N)[(N-1):]
                 def gaus(self,xSamples,Amp,Mu,Sigma):
                     return ( Amp / ((2*numpy.pi)**0.5 * Sigma) ) * numpy.exp( -( xSamples - Mu )**2 / ( 2 * (Sigma**2) ))
                 def Moments(self, ySamples, xSamples):
                     Pot = numpy.nansum( ySamples )                              # Potencia, momento 0
                     yNorm = ySamples / Pot
                     Vr = numpy.nansum( yNorm * xSamples )                       # Velocidad radial, mu, corrimiento doppler, primer momento
                     Sigma2 = abs(numpy.nansum( yNorm * ( xSamples - Vr )**2 ))  # Segundo Momento
                     Desv = Sigma2**0.5                                          # Desv. Estandar, Ancho espectral
                     return numpy.array([Pot, Vr, Desv])
                 def WindEstimation(self, spc, cspc, pairsList, ChanDist, Height, noise, AbbsisaRange, dbSNR, SNRlimit):
                     ySamples=numpy.ones([spc.shape[0],spc.shape[1]])
                     phase=numpy.ones([spc.shape[0],spc.shape[1]])
                     CSPCSamples=numpy.ones([spc.shape[0],spc.shape[1]],dtype=numpy.complex_)
                     coherence=numpy.ones([spc.shape[0],spc.shape[1]])
                     PhaseSlope=numpy.zeros(spc.shape[0])
                     PhaseInter=numpy.ones(spc.shape[0])
                     xFrec=AbbsisaRange[0][0:spc.shape[1]]
                     xVel =AbbsisaRange[2][0:spc.shape[1]]
                     Vv=numpy.empty(spc.shape[2])*0
                     SPCav = numpy.average(spc, axis=0)-numpy.average(noise) #spc[0]-noise[0]#
                     SPCmoments = self.Moments(SPCav[:,Height], xVel )
                     CSPCmoments = []
                     cspcNoise = numpy.empty(3)
                     '''Getting Eij and Nij'''
                     Xi01=ChanDist[0][0]
                     Eta01=ChanDist[0][1]
                     Xi02=ChanDist[1][0]
                     Eta02=ChanDist[1][1]
                     Xi12=ChanDist[2][0]
                     Eta12=ChanDist[2][1]
                     z = spc.copy()
                     z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
                     for i in range(spc.shape[0]):
                         '''****** Line of Data SPC ******'''
                         zline=z[i,:,Height].copy() - noise[i]   # Se resta ruido
                         '''****** SPC is normalized ******'''
                         SmoothSPC =self.moving_average(zline.copy(),N=1)     # Se suaviza el ruido
                         FactNorm = SmoothSPC/numpy.nansum(SmoothSPC)         # SPC Normalizado y suavizado
                         xSamples = xFrec                   # Se toma el rango de frecuncias
                         ySamples[i] = FactNorm             # Se toman los valores de SPC normalizado
                     for i in range(spc.shape[0]):
                         '''****** Line of Data CSPC ******'''
                         cspcLine = ( cspc[i,:,Height].copy())# - noise[i] )     # no! Se resta el ruido
                         SmoothCSPC =self.moving_average(cspcLine,N=1)         # Se suaviza el ruido
                         cspcNorm = SmoothCSPC/numpy.nansum(SmoothCSPC)        # CSPC normalizado y suavizado
                         '''****** CSPC is normalized with respect to Briggs and Vincent ******'''
                         chan_index0 = pairsList[i][0]
                         chan_index1 = pairsList[i][1]
                         CSPCFactor= numpy.abs(numpy.nansum(ySamples[chan_index0]))**2  *  numpy.abs(numpy.nansum(ySamples[chan_index1]))**2
                         CSPCNorm = cspcNorm / numpy.sqrt(CSPCFactor)
                         CSPCSamples[i] = CSPCNorm
                         coherence[i] = numpy.abs(CSPCSamples[i]) / numpy.sqrt(CSPCFactor)
                         #coherence[i]= self.moving_average(coherence[i],N=1)
                         phase[i] = self.moving_average( numpy.arctan2(CSPCSamples[i].imag, CSPCSamples[i].real),N=1)#*180/numpy.pi
                     CSPCmoments = numpy.vstack([self.Moments(numpy.abs(CSPCSamples[0]), xSamples),
                                                 self.Moments(numpy.abs(CSPCSamples[1]), xSamples),
                                                 self.Moments(numpy.abs(CSPCSamples[2]), xSamples)])
                     popt=[1e-10,0,1e-10]
                     popt01, popt02, popt12 = [1e-10,1e-10,1e-10], [1e-10,1e-10,1e-10] ,[1e-10,1e-10,1e-10]
                     FitGauss01, FitGauss02, FitGauss12 = numpy.empty(len(xSamples))*0, numpy.empty(len(xSamples))*0, numpy.empty(len(xSamples))*0
                     CSPCMask01 = numpy.abs(CSPCSamples[0])
                     CSPCMask02 = numpy.abs(CSPCSamples[1])
                     CSPCMask12 = numpy.abs(CSPCSamples[2])
                     mask01 = ~numpy.isnan(CSPCMask01)
                     mask02 = ~numpy.isnan(CSPCMask02)
                     mask12 = ~numpy.isnan(CSPCMask12)
                     #mask = ~numpy.isnan(CSPCMask01)
                     CSPCMask01 = CSPCMask01[mask01]
                     CSPCMask02 = CSPCMask02[mask02]
                     CSPCMask12 = CSPCMask12[mask12]
                     #CSPCMask01 = numpy.ma.masked_invalid(CSPCMask01)
                     '''***Fit Gauss CSPC01***'''
                     if dbSNR > SNRlimit and numpy.abs(SPCmoments[1])<3 :
                         try:
                             popt01,pcov = curve_fit(self.gaus,xSamples[mask01],numpy.abs(CSPCMask01),p0=CSPCmoments[0])
                             popt02,pcov = curve_fit(self.gaus,xSamples[mask02],numpy.abs(CSPCMask02),p0=CSPCmoments[1])
                             popt12,pcov = curve_fit(self.gaus,xSamples[mask12],numpy.abs(CSPCMask12),p0=CSPCmoments[2])
                             FitGauss01 = self.gaus(xSamples,*popt01)
                             FitGauss02 = self.gaus(xSamples,*popt02)
                             FitGauss12 = self.gaus(xSamples,*popt12)
                         except:
                             FitGauss01=numpy.ones(len(xSamples))*numpy.mean(numpy.abs(CSPCSamples[0]))
                             FitGauss02=numpy.ones(len(xSamples))*numpy.mean(numpy.abs(CSPCSamples[1]))
                             FitGauss12=numpy.ones(len(xSamples))*numpy.mean(numpy.abs(CSPCSamples[2]))
                     CSPCopt = numpy.vstack([popt01,popt02,popt12])
                     '''****** Getting fij width ******'''
                     yMean = numpy.average(ySamples, axis=0)  # ySamples[0]
                     '''******* Getting fitting Gaussian *******'''
                     meanGauss = sum(xSamples*yMean) / len(xSamples)              # Mu, velocidad radial (frecuencia)
                     sigma2 = sum(yMean*(xSamples-meanGauss)**2) / len(xSamples)  # Varianza, Ancho espectral (frecuencia)
                     yMoments = self.Moments(yMean, xSamples)
                     if dbSNR > SNRlimit and numpy.abs(SPCmoments[1])<3: # and abs(meanGauss/sigma2) > 0.00001:
                         try:
                             popt,pcov = curve_fit(self.gaus,xSamples,yMean,p0=yMoments)
                             FitGauss=self.gaus(xSamples,*popt)
                         except :#RuntimeError:
                             FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean)
                     else:
                         FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean)
                     '''****** Getting Fij ******'''
                     Fijcspc = CSPCopt[:,2]/2*3
                     GaussCenter = popt[1] #xFrec[GCpos]
                     #Punto en Eje X de la Gaussiana donde se encuentra el centro
                     ClosestCenter = xSamples[numpy.abs(xSamples-GaussCenter).argmin()]
                     PointGauCenter = numpy.where(xSamples==ClosestCenter)[0][0]
                     #Punto e^-1 hubicado en la Gaussiana
                     PeMinus1 = numpy.max(FitGauss)* numpy.exp(-1)
                     FijClosest = FitGauss[numpy.abs(FitGauss-PeMinus1).argmin()] # El punto mas cercano a "Peminus1" dentro de "FitGauss"
                     PointFij = numpy.where(FitGauss==FijClosest)[0][0]
                     if xSamples[PointFij] > xSamples[PointGauCenter]:
                         Fij = xSamples[PointFij] - xSamples[PointGauCenter]
                     else:
                         Fij = xSamples[PointGauCenter] - xSamples[PointFij]
                     '''****** Taking frequency ranges from SPCs ******'''
                     #GaussCenter = popt[1]     #Primer momento 01
                     GauWidth = popt[2] *3/2        #Ancho de banda de Gau01
                     Range = numpy.empty(2)
                     Range[0] = GaussCenter - GauWidth
                     Range[1] = GaussCenter + GauWidth
                     #Punto en Eje X de la Gaussiana donde se encuentra ancho de banda (min:max)
                     ClosRangeMin = xSamples[numpy.abs(xSamples-Range[0]).argmin()]
                     ClosRangeMax = xSamples[numpy.abs(xSamples-Range[1]).argmin()]
                     PointRangeMin = numpy.where(xSamples==ClosRangeMin)[0][0]
                     PointRangeMax = numpy.where(xSamples==ClosRangeMax)[0][0]
                     Range=numpy.array([ PointRangeMin, PointRangeMax ])
                     FrecRange = xFrec[ Range[0] : Range[1] ]
                     VelRange  = xVel[ Range[0] : Range[1] ]
                     '''****** Getting SCPC Slope ******'''
                     for i in range(spc.shape[0]):
                         if len(FrecRange)>5 and len(FrecRange)<spc.shape[1]*0.3:
                             PhaseRange=self.moving_average(phase[i,Range[0]:Range[1]],N=3)
                             '''***********************VelRange******************'''
                             mask = ~numpy.isnan(FrecRange) & ~numpy.isnan(PhaseRange)
                             if len(FrecRange) == len(PhaseRange):
                                 try:
                                     slope, intercept, r_value, p_value, std_err = stats.linregress(FrecRange[mask], PhaseRange[mask])
                                     PhaseSlope[i]=slope
                                     PhaseInter[i]=intercept
                                 except:
                                     PhaseSlope[i]=0
                                     PhaseInter[i]=0
                             else:
                                 PhaseSlope[i]=0
                                 PhaseInter[i]=0
                         else:
                             PhaseSlope[i]=0
                             PhaseInter[i]=0
                         '''Getting constant C'''
                         cC=(Fij*numpy.pi)**2
                         '''****** Getting constants F and G ******'''
                         MijEijNij=numpy.array([[Xi02,Eta02], [Xi12,Eta12]])
                         MijResult0=(-PhaseSlope[1]*cC) / (2*numpy.pi)
                         MijResult1=(-PhaseSlope[2]*cC) / (2*numpy.pi)
                         MijResults=numpy.array([MijResult0,MijResult1])
                         (cF,cG) = numpy.linalg.solve(MijEijNij, MijResults)
                         '''****** Getting constants A, B and H ******'''
                         W01=numpy.nanmax( FitGauss01 ) #numpy.abs(CSPCSamples[0]))
                         W02=numpy.nanmax( FitGauss02 ) #numpy.abs(CSPCSamples[1]))
                         W12=numpy.nanmax( FitGauss12 ) #numpy.abs(CSPCSamples[2]))
                         WijResult0=((cF*Xi01+cG*Eta01)**2)/cC - numpy.log(W01 / numpy.sqrt(numpy.pi/cC))
                         WijResult1=((cF*Xi02+cG*Eta02)**2)/cC - numpy.log(W02 / numpy.sqrt(numpy.pi/cC))
                         WijResult2=((cF*Xi12+cG*Eta12)**2)/cC - numpy.log(W12 / numpy.sqrt(numpy.pi/cC))
                         WijResults=numpy.array([WijResult0, WijResult1, WijResult2])
                         WijEijNij=numpy.array([ [Xi01**2, Eta01**2, 2*Xi01*Eta01] , [Xi02**2, Eta02**2, 2*Xi02*Eta02] , [Xi12**2, Eta12**2, 2*Xi12*Eta12] ])
                         (cA,cB,cH) = numpy.linalg.solve(WijEijNij, WijResults)
                         VxVy=numpy.array([[cA,cH],[cH,cB]])
                         VxVyResults=numpy.array([-cF,-cG])
                         (Vx,Vy) = numpy.linalg.solve(VxVy, VxVyResults)
                         Vzon = Vy
                         Vmer = Vx
                         Vmag=numpy.sqrt(Vzon**2+Vmer**2)
                         Vang=numpy.arctan2(Vmer,Vzon)
                         if numpy.abs( popt[1] ) < 3.5 and len(FrecRange)>4:
                             Vver=popt[1]
                         else:
                             Vver=numpy.NaN
                         FitGaussCSPC = numpy.array([FitGauss01,FitGauss02,FitGauss12])
                     return Vzon, Vmer, Vver, GaussCenter, PhaseSlope, FitGaussCSPC
             class SpectralMoments(Operation):
                 '''
                     Function SpectralMoments()
                     Calculates moments (power, mean, standard deviation) and SNR of the signal
                     Type of dataIn:    Spectra
                     Configuration Parameters:
                         dirCosx    :     Cosine director in X axis
                         dirCosy    :     Cosine director in Y axis
                         elevation  :
                         azimuth    :
                     Input:
                         channelList    :    simple channel list to select e.g. [2,3,7]
                         self.dataOut.data_pre        :    Spectral data
                         self.dataOut.abscissaList    :    List of frequencies
                         self.dataOut.noise           :    Noise level per channel
                     Affected:
                         self.dataOut.moments        :    Parameters per channel
                         self.dataOut.data_SNR       :    SNR per channel
                 '''
                 def run(self, dataOut):
                     #dataOut.data_pre = dataOut.data_pre[0]
                     data = dataOut.data_pre[0]
                     absc = dataOut.abscissaList[:-1]
                     noise = dataOut.noise
                     nChannel = data.shape[0]
                     data_param = numpy.zeros((nChannel, 4, data.shape[2]))
                     for ind in range(nChannel):
                         data_param[ind,:,:] = self.__calculateMoments( data[ind,:,:] , absc , noise[ind] )
                     dataOut.moments = data_param[:,1:,:]
                     dataOut.data_SNR = data_param[:,0]
                     dataOut.data_POW = data_param[:,1]
                     dataOut.data_DOP = data_param[:,2]
                     dataOut.data_WIDTH = data_param[:,3]
                     return dataOut
                 def __calculateMoments(self, oldspec, oldfreq, n0,
                                        nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None):
                     if (nicoh is None): nicoh = 1
                     if (graph is None): graph = 0
                     if (smooth is None): smooth = 0
                     elif (self.smooth < 3): smooth = 0
                     if (type1 is None): type1 = 0
                     if (fwindow is None): fwindow = numpy.zeros(oldfreq.size) + 1
                     if (snrth is None): snrth = -3
                     if (dc is None): dc = 0
                     if (aliasing is None): aliasing = 0
                     if (oldfd is None): oldfd = 0
                     if (wwauto is None): wwauto = 0
                     if (n0 < 1.e-20):   n0 = 1.e-20
                     freq = oldfreq
                     vec_power = numpy.zeros(oldspec.shape[1])
                     vec_fd = numpy.zeros(oldspec.shape[1])
                     vec_w = numpy.zeros(oldspec.shape[1])
                     vec_snr = numpy.zeros(oldspec.shape[1])
                     oldspec = numpy.ma.masked_invalid(oldspec)
                     for ind in range(oldspec.shape[1]):
                         spec = oldspec[:,ind]
                         aux = spec*fwindow
                         max_spec = aux.max()
                         m = list(aux).index(max_spec)
                         #Smooth
                         if (smooth == 0):   spec2 = spec
                         else:   spec2 = scipy.ndimage.filters.uniform_filter1d(spec,size=smooth)
                         #    Calculo de Momentos
                         bb = spec2[list(range(m,spec2.size))]
                         bb = (bb<n0).nonzero()
                         bb = bb[0]
                         ss = spec2[list(range(0,m + 1))]
                         ss = (ss<n0).nonzero()
                         ss = ss[0]
                         if (bb.size == 0):
                             bb0 = spec.size - 1 - m
                         else:
                             bb0 = bb[0] - 1
                             if (bb0 < 0):
                                 bb0 = 0
                         if (ss.size == 0):   ss1 = 1
                         else: ss1 = max(ss) + 1
                         if (ss1 > m):   ss1 = m
                         valid = numpy.asarray(list(range(int(m + bb0 - ss1 + 1)))) + ss1
                         power = ((spec2[valid] - n0)*fwindow[valid]).sum()
                         fd = ((spec2[valid]- n0)*freq[valid]*fwindow[valid]).sum()/power
                         w = math.sqrt(((spec2[valid] - n0)*fwindow[valid]*(freq[valid]- fd)**2).sum()/power)
                         snr = (spec2.mean()-n0)/n0
                         if (snr < 1.e-20) :
                             snr = 1.e-20
                         vec_power[ind] = power
                         vec_fd[ind] = fd
                         vec_w[ind] = w
                         vec_snr[ind] = snr
                     moments = numpy.vstack((vec_snr, vec_power, vec_fd, vec_w))
                     return moments
                 #------------------    Get SA Parameters    --------------------------
                 def GetSAParameters(self):
                     #SA en frecuencia
                     pairslist = self.dataOut.groupList
                     num_pairs = len(pairslist)
                     vel = self.dataOut.abscissaList
                     spectra = self.dataOut.data_pre
                     cspectra = self.dataIn.data_cspc
                     delta_v = vel[1] - vel[0]
                     #Calculating the power spectrum
                     spc_pow = numpy.sum(spectra, 3)*delta_v
                     #Normalizing Spectra
                     norm_spectra = spectra/spc_pow
                     #Calculating the norm_spectra at peak
                     max_spectra = numpy.max(norm_spectra, 3)
                     #Normalizing Cross Spectra
                     norm_cspectra = numpy.zeros(cspectra.shape)
                     for i in range(num_chan):
                         norm_cspectra[i,:,:] = cspectra[i,:,:]/numpy.sqrt(spc_pow[pairslist[i][0],:]*spc_pow[pairslist[i][1],:])
                     max_cspectra = numpy.max(norm_cspectra,2)
                     max_cspectra_index = numpy.argmax(norm_cspectra, 2)
                     for i in range(num_pairs):
                         cspc_par[i,:,:] = __calculateMoments(norm_cspectra)
                 #-------------------    Get Lags    ----------------------------------
             class SALags(Operation):
                 '''
                 Function GetMoments()
                 Input:
                     self.dataOut.data_pre
                     self.dataOut.abscissaList
                     self.dataOut.noise
                     self.dataOut.normFactor
                     self.dataOut.data_SNR
                     self.dataOut.groupList
                     self.dataOut.nChannels
                 Affected:
                     self.dataOut.data_param
                 '''
                 def run(self, dataOut):
                     data_acf = dataOut.data_pre[0]
                     data_ccf = dataOut.data_pre[1]
                     normFactor_acf = dataOut.normFactor[0]
                     normFactor_ccf = dataOut.normFactor[1]
                     pairs_acf = dataOut.groupList[0]
                     pairs_ccf = dataOut.groupList[1]
                     nHeights = dataOut.nHeights
                     absc = dataOut.abscissaList
                     noise = dataOut.noise
                     SNR = dataOut.data_SNR
                     nChannels = dataOut.nChannels
             #         pairsList = dataOut.groupList
             #         pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairsList, nChannels)
                     for l in range(len(pairs_acf)):
                         data_acf[l,:,:] = data_acf[l,:,:]/normFactor_acf[l,:]
                     for l in range(len(pairs_ccf)):
                         data_ccf[l,:,:] = data_ccf[l,:,:]/normFactor_ccf[l,:]
                     dataOut.data_param = numpy.zeros((len(pairs_ccf)*2 + 1, nHeights))
                     dataOut.data_param[:-1,:] = self.__calculateTaus(data_acf, data_ccf, absc)
                     dataOut.data_param[-1,:] = self.__calculateLag1Phase(data_acf, absc)
                     return
             #     def __getPairsAutoCorr(self, pairsList, nChannels):
             #
             #         pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
             #
             #         for l in range(len(pairsList)):
             #             firstChannel = pairsList[l][0]
             #             secondChannel = pairsList[l][1]
             #
             #             #Obteniendo pares de Autocorrelacion
             #             if firstChannel == secondChannel:
             #                 pairsAutoCorr[firstChannel] = int(l)
             #
             #         pairsAutoCorr = pairsAutoCorr.astype(int)
             #
             #         pairsCrossCorr = range(len(pairsList))
             #         pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
             #
             #         return pairsAutoCorr, pairsCrossCorr
                 def __calculateTaus(self, data_acf, data_ccf, lagRange):
                     lag0 = data_acf.shape[1]/2
                     #Funcion de Autocorrelacion
                     mean_acf = stats.nanmean(data_acf, axis = 0)
                     #Obtencion Indice de TauCross
                     ind_ccf = data_ccf.argmax(axis = 1)
                     #Obtencion Indice de TauAuto
                     ind_acf = numpy.zeros(ind_ccf.shape,dtype = 'int')
                     ccf_lag0 = data_ccf[:,lag0,:]
                     for i in range(ccf_lag0.shape[0]):
                         ind_acf[i,:] = numpy.abs(mean_acf - ccf_lag0[i,:]).argmin(axis = 0)
                     #Obtencion de TauCross y TauAuto
                     tau_ccf = lagRange[ind_ccf]
                     tau_acf  = lagRange[ind_acf]
                     Nan1, Nan2 = numpy.where(tau_ccf == lagRange[0])
                     tau_ccf[Nan1,Nan2] = numpy.nan
                     tau_acf[Nan1,Nan2] = numpy.nan
                     tau = numpy.vstack((tau_ccf,tau_acf))
                     return tau
                 def __calculateLag1Phase(self, data, lagTRange):
                     data1 = stats.nanmean(data, axis = 0)
                     lag1 = numpy.where(lagTRange == 0)[0][0] + 1
                     phase = numpy.angle(data1[lag1,:])
                     return phase
             class SpectralFitting(Operation):
                 '''
                     Function GetMoments()
                     Input:
                     Output:
                     Variables modified:
                 '''
                 def run(self, dataOut, getSNR = True, path=None, file=None, groupList=None):
                     if path != None:
                         sys.path.append(path)
                     self.dataOut.library = importlib.import_module(file)
                     #To be inserted as a parameter
                     groupArray = numpy.array(groupList)
             #         groupArray = numpy.array([[0,1],[2,3]])
                     self.dataOut.groupList = groupArray
                     nGroups = groupArray.shape[0]
                     nChannels = self.dataIn.nChannels
                     nHeights=self.dataIn.heightList.size
                     #Parameters Array
                     self.dataOut.data_param = None
                     #Set constants
                     constants = self.dataOut.library.setConstants(self.dataIn)
                     self.dataOut.constants = constants
                     M = self.dataIn.normFactor
                     N = self.dataIn.nFFTPoints
                     ippSeconds = self.dataIn.ippSeconds
                     K = self.dataIn.nIncohInt
                     pairsArray = numpy.array(self.dataIn.pairsList)
                     #List of possible combinations
                     listComb = itertools.combinations(numpy.arange(groupArray.shape[1]),2)
                     indCross = numpy.zeros(len(list(listComb)), dtype = 'int')
                     if getSNR:
                         listChannels = groupArray.reshape((groupArray.size))
                         listChannels.sort()
                         noise = self.dataIn.getNoise()
                         self.dataOut.data_SNR = self.__getSNR(self.dataIn.data_spc[listChannels,:,:], noise[listChannels])
                     for i in range(nGroups):
                         coord = groupArray[i,:]
                         #Input data array
                         data = self.dataIn.data_spc[coord,:,:]/(M*N)
                         data = data.reshape((data.shape[0]*data.shape[1],data.shape[2]))
                         #Cross Spectra data array for Covariance Matrixes
                         ind = 0
                         for pairs in listComb:
                             pairsSel = numpy.array([coord[x],coord[y]])
                             indCross[ind] = int(numpy.where(numpy.all(pairsArray == pairsSel, axis = 1))[0][0])
                             ind += 1
                         dataCross = self.dataIn.data_cspc[indCross,:,:]/(M*N)
                         dataCross = dataCross**2/K
                         for h in range(nHeights):
                             #Input
                             d = data[:,h]
                             #Covariance Matrix
                             D = numpy.diag(d**2/K)
                             ind = 0
                             for pairs in listComb:
                                 #Coordinates in Covariance Matrix
                                 x = pairs[0]
                                 y = pairs[1]
                                 #Channel Index
                                 S12 = dataCross[ind,:,h]
                                 D12 = numpy.diag(S12)
                                 #Completing Covariance Matrix with Cross Spectras
                                 D[x*N:(x+1)*N,y*N:(y+1)*N] = D12
                                 D[y*N:(y+1)*N,x*N:(x+1)*N] = D12
                                 ind += 1
                             Dinv=numpy.linalg.inv(D)
                             L=numpy.linalg.cholesky(Dinv)
                             LT=L.T
                             dp = numpy.dot(LT,d)
                             #Initial values
                             data_spc = self.dataIn.data_spc[coord,:,h]
                             if (h>0)and(error1[3]<5):
                                 p0 = self.dataOut.data_param[i,:,h-1]
                             else:
                                 p0 = numpy.array(self.dataOut.library.initialValuesFunction(data_spc, constants, i))
                             try:
                                 #Least Squares
                                 minp,covp,infodict,mesg,ier = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants),full_output=True)
             #                   minp,covp = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants))
                                 #Chi square error
                                 error0 = numpy.sum(infodict['fvec']**2)/(2*N)
                                 #Error with Jacobian
                                 error1 = self.dataOut.library.errorFunction(minp,constants,LT)
                             except:
                                 minp = p0*numpy.nan
                                 error0 = numpy.nan
                                 error1 = p0*numpy.nan
                             #Save
                             if self.dataOut.data_param is None:
                                 self.dataOut.data_param = numpy.zeros((nGroups, p0.size, nHeights))*numpy.nan
                                 self.dataOut.data_error = numpy.zeros((nGroups, p0.size + 1, nHeights))*numpy.nan
                             self.dataOut.data_error[i,:,h] = numpy.hstack((error0,error1))
                             self.dataOut.data_param[i,:,h] = minp
                     return
                 def __residFunction(self, p, dp, LT, constants):
                     fm = self.dataOut.library.modelFunction(p, constants)
                     fmp=numpy.dot(LT,fm)
                     return  dp-fmp
                 def __getSNR(self, z, noise):
                     avg = numpy.average(z, axis=1)
                     SNR = (avg.T-noise)/noise
                     SNR = SNR.T
                     return SNR
                 def __chisq(p,chindex,hindex):
                     #similar to Resid but calculates CHI**2
                     [LT,d,fm]=setupLTdfm(p,chindex,hindex)
                     dp=numpy.dot(LT,d)
                     fmp=numpy.dot(LT,fm)
                     chisq=numpy.dot((dp-fmp).T,(dp-fmp))
                     return chisq
             class WindProfiler(Operation):
                 __isConfig = False
                 __initime = None
                 __lastdatatime = None
                 __integrationtime = None
                 __buffer = None
                 __dataReady = False
                 __firstdata = None
                 n = None
                 def __init__(self):
                     Operation.__init__(self)
                 def __calculateCosDir(self, elev, azim):
                     zen = (90 - elev)*numpy.pi/180
                     azim = azim*numpy.pi/180
                     cosDirX = numpy.sqrt((1-numpy.cos(zen)**2)/((1+numpy.tan(azim)**2)))
                     cosDirY = numpy.sqrt(1-numpy.cos(zen)**2-cosDirX**2)
                     signX = numpy.sign(numpy.cos(azim))
                     signY = numpy.sign(numpy.sin(azim))
                     cosDirX = numpy.copysign(cosDirX, signX)
                     cosDirY = numpy.copysign(cosDirY, signY)
                     return cosDirX, cosDirY
                 def __calculateAngles(self, theta_x, theta_y, azimuth):
                     dir_cosw = numpy.sqrt(1-theta_x**2-theta_y**2)
                     zenith_arr = numpy.arccos(dir_cosw)
                     azimuth_arr = numpy.arctan2(theta_x,theta_y) + azimuth*math.pi/180
                     dir_cosu = numpy.sin(azimuth_arr)*numpy.sin(zenith_arr)
                     dir_cosv = numpy.cos(azimuth_arr)*numpy.sin(zenith_arr)
                     return azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw
                 def __calculateMatA(self, dir_cosu, dir_cosv, dir_cosw, horOnly):
             #
                     if horOnly:
                         A = numpy.c_[dir_cosu,dir_cosv]
                     else:
                         A = numpy.c_[dir_cosu,dir_cosv,dir_cosw]
                     A = numpy.asmatrix(A)
                     A1 = numpy.linalg.inv(A.transpose()*A)*A.transpose()
                     return A1
                 def __correctValues(self, heiRang, phi, velRadial, SNR):
                     listPhi = phi.tolist()
                     maxid = listPhi.index(max(listPhi))
                     minid = listPhi.index(min(listPhi))
                     rango = list(range(len(phi)))
                #     rango = numpy.delete(rango,maxid)
                     heiRang1 = heiRang*math.cos(phi[maxid])
                     heiRangAux = heiRang*math.cos(phi[minid])
                     indOut = (heiRang1 < heiRangAux[0]).nonzero()
                     heiRang1 = numpy.delete(heiRang1,indOut)
                     velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
                     SNR1 = numpy.zeros([len(phi),len(heiRang1)])
                     for i in rango:
                         x = heiRang*math.cos(phi[i])
                         y1 = velRadial[i,:]
                         f1 = interpolate.interp1d(x,y1,kind = 'cubic')
                         x1 = heiRang1
                         y11 = f1(x1)
                         y2 = SNR[i,:]
                         f2 = interpolate.interp1d(x,y2,kind = 'cubic')
                         y21 = f2(x1)
                         velRadial1[i,:] = y11
                         SNR1[i,:] = y21
                     return heiRang1, velRadial1, SNR1
                 def __calculateVelUVW(self, A, velRadial):
                     #Operacion Matricial
             #         velUVW = numpy.zeros((velRadial.shape[1],3))
             #         for ind in range(velRadial.shape[1]):
             #             velUVW[ind,:] = numpy.dot(A,velRadial[:,ind])
             #         velUVW = velUVW.transpose()
                     velUVW = numpy.zeros((A.shape[0],velRadial.shape[1]))
                     velUVW[:,:] = numpy.dot(A,velRadial)
                     return velUVW
             #     def techniqueDBS(self, velRadial0, dirCosx, disrCosy, azimuth, correct, horizontalOnly, heiRang, SNR0):
                 def techniqueDBS(self, kwargs):
                     """
                     Function that implements Doppler Beam Swinging (DBS) technique.
                     Input:    Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
                                 Direction correction (if necessary), Ranges and SNR
                     Output:    Winds estimation (Zonal, Meridional and Vertical)
                     Parameters affected:    Winds, height range, SNR
                     """
                     velRadial0 = kwargs['velRadial']
                     heiRang = kwargs['heightList']
                     SNR0 = kwargs['SNR']
                     if 'dirCosx' in kwargs and 'dirCosy' in kwargs:
                         theta_x = numpy.array(kwargs['dirCosx'])
                         theta_y = numpy.array(kwargs['dirCosy'])
                     else:
                         elev = numpy.array(kwargs['elevation'])
                         azim = numpy.array(kwargs['azimuth'])
                         theta_x, theta_y = self.__calculateCosDir(elev, azim)
                     azimuth = kwargs['correctAzimuth']
                     if 'horizontalOnly' in kwargs:
                         horizontalOnly = kwargs['horizontalOnly']
                     else:   horizontalOnly = False
                     if 'correctFactor' in kwargs:
                         correctFactor = kwargs['correctFactor']
                     else:   correctFactor = 1
                     if 'channelList' in kwargs:
                         channelList = kwargs['channelList']
                         if len(channelList) == 2:
                             horizontalOnly = True
                         arrayChannel = numpy.array(channelList)
                         param = param[arrayChannel,:,:]
                         theta_x = theta_x[arrayChannel]
                         theta_y = theta_y[arrayChannel]
                     azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw = self.__calculateAngles(theta_x, theta_y, azimuth)
                     heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, zenith_arr, correctFactor*velRadial0, SNR0)
                     A = self.__calculateMatA(dir_cosu, dir_cosv, dir_cosw, horizontalOnly)
                     #Calculo de Componentes de la velocidad con DBS
                     winds = self.__calculateVelUVW(A,velRadial1)
                     return winds, heiRang1, SNR1
                 def __calculateDistance(self, posx, posy, pairs_ccf, azimuth = None):
                     nPairs = len(pairs_ccf)
                     posx = numpy.asarray(posx)
                     posy = numpy.asarray(posy)
                     #Rotacion Inversa para alinear con el azimuth
                     if azimuth!= None:
                         azimuth = azimuth*math.pi/180
                         posx1 = posx*math.cos(azimuth) + posy*math.sin(azimuth)
                         posy1 = -posx*math.sin(azimuth) + posy*math.cos(azimuth)
                     else:
                         posx1 = posx
                         posy1 = posy
                     #Calculo de Distancias
                     distx = numpy.zeros(nPairs)
                     disty = numpy.zeros(nPairs)
                     dist = numpy.zeros(nPairs)
                     ang = numpy.zeros(nPairs)
                     for i in range(nPairs):
                         distx[i] = posx1[pairs_ccf[i][1]] - posx1[pairs_ccf[i][0]]
                         disty[i] = posy1[pairs_ccf[i][1]] - posy1[pairs_ccf[i][0]]
                         dist[i] = numpy.sqrt(distx[i]**2 + disty[i]**2)
                         ang[i] = numpy.arctan2(disty[i],distx[i])
                     return distx, disty, dist, ang
                     #Calculo de Matrices
             #         nPairs = len(pairs)
             #         ang1 = numpy.zeros((nPairs, 2, 1))
             #         dist1 = numpy.zeros((nPairs, 2, 1))
             #
             #         for j in range(nPairs):
             #             dist1[j,0,0] = dist[pairs[j][0]]
             #             dist1[j,1,0] = dist[pairs[j][1]]
             #             ang1[j,0,0] = ang[pairs[j][0]]
             #             ang1[j,1,0] = ang[pairs[j][1]]
             #
             #         return distx,disty, dist1,ang1
                 def __calculateVelVer(self, phase, lagTRange, _lambda):
                     Ts = lagTRange[1] - lagTRange[0]
                     velW = -_lambda*phase/(4*math.pi*Ts)
                     return velW
                 def __calculateVelHorDir(self, dist, tau1, tau2, ang):
                     nPairs = tau1.shape[0]
                     nHeights = tau1.shape[1]
                     vel = numpy.zeros((nPairs,3,nHeights))
                     dist1 = numpy.reshape(dist, (dist.size,1))
                     angCos = numpy.cos(ang)
                     angSin = numpy.sin(ang)
                     vel0 = dist1*tau1/(2*tau2**2)
                     vel[:,0,:] = (vel0*angCos).sum(axis = 1)
                     vel[:,1,:] = (vel0*angSin).sum(axis = 1)
                     ind = numpy.where(numpy.isinf(vel))
                     vel[ind] = numpy.nan
                     return vel
             #     def __getPairsAutoCorr(self, pairsList, nChannels):
             #
             #         pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
             #
             #         for l in range(len(pairsList)):
             #             firstChannel = pairsList[l][0]
             #             secondChannel = pairsList[l][1]
             #
             #             #Obteniendo pares de Autocorrelacion
             #             if firstChannel == secondChannel:
             #                 pairsAutoCorr[firstChannel] = int(l)
             #
             #         pairsAutoCorr = pairsAutoCorr.astype(int)
             #
             #         pairsCrossCorr = range(len(pairsList))
             #         pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
             #
             #         return pairsAutoCorr, pairsCrossCorr
             #     def techniqueSA(self, pairsSelected, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, lagTRange, correctFactor):
                 def techniqueSA(self, kwargs):
                     """
                     Function that implements Spaced Antenna (SA) technique.
                     Input:    Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
                                 Direction correction (if necessary), Ranges and SNR
                     Output:    Winds estimation (Zonal, Meridional and Vertical)
                     Parameters affected:    Winds
                     """
                     position_x = kwargs['positionX']
                     position_y = kwargs['positionY']
                     azimuth = kwargs['azimuth']
                     if 'correctFactor' in kwargs:
                         correctFactor = kwargs['correctFactor']
                     else:
                         correctFactor = 1
                     groupList = kwargs['groupList']
                     pairs_ccf = groupList[1]
                     tau = kwargs['tau']
                     _lambda = kwargs['_lambda']
                     #Cross Correlation pairs obtained
             #         pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairssList, nChannels)
             #         pairsArray = numpy.array(pairsList)[pairsCrossCorr]
             #         pairsSelArray = numpy.array(pairsSelected)
             #         pairs = []
             #
             #         #Wind estimation pairs obtained
             #         for i in range(pairsSelArray.shape[0]/2):
             #             ind1 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i], axis = 1))[0][0]
             #             ind2 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i + 1], axis = 1))[0][0]
             #             pairs.append((ind1,ind2))
                     indtau = tau.shape[0]/2
                     tau1 = tau[:indtau,:]
                     tau2 = tau[indtau:-1,:]
             #         tau1 = tau1[pairs,:]
             #         tau2 = tau2[pairs,:]
                     phase1 = tau[-1,:]
                     #---------------------------------------------------------------------
                     #Metodo Directo
                     distx, disty, dist, ang = self.__calculateDistance(position_x, position_y, pairs_ccf,azimuth)
                     winds = self.__calculateVelHorDir(dist, tau1, tau2, ang)
                     winds = stats.nanmean(winds, axis=0)
                     #---------------------------------------------------------------------
                     #Metodo General
             #         distx, disty, dist = self.calculateDistance(position_x,position_y,pairsCrossCorr, pairsList, azimuth)
             #         #Calculo Coeficientes de Funcion de Correlacion
             #         F,G,A,B,H = self.calculateCoef(tau1,tau2,distx,disty,n)
             #         #Calculo de Velocidades
             #         winds = self.calculateVelUV(F,G,A,B,H)
                     #---------------------------------------------------------------------
                     winds[2,:] = self.__calculateVelVer(phase1, lagTRange, _lambda)
                     winds = correctFactor*winds
                     return winds
                 def __checkTime(self, currentTime, paramInterval, outputInterval):
                     dataTime = currentTime + paramInterval
                     deltaTime = dataTime - self.__initime
                     if deltaTime >= outputInterval or deltaTime < 0:
                         self.__dataReady = True
                     return
                 def techniqueMeteors(self, arrayMeteor, meteorThresh, heightMin, heightMax):
                     '''
                     Function that implements winds estimation technique with detected meteors.
                     Input:    Detected meteors, Minimum meteor quantity to wind estimation
                     Output:    Winds estimation (Zonal and Meridional)
                     Parameters affected:    Winds
                     '''
                     #Settings
                     nInt = (heightMax - heightMin)/2
                     nInt = int(nInt)
                     winds = numpy.zeros((2,nInt))*numpy.nan
                     #Filter errors
                     error = numpy.where(arrayMeteor[:,-1] == 0)[0]
                     finalMeteor = arrayMeteor[error,:]
                     #Meteor Histogram
                     finalHeights = finalMeteor[:,2]
                     hist = numpy.histogram(finalHeights, bins = nInt, range = (heightMin,heightMax))
                     nMeteorsPerI = hist[0]
                     heightPerI = hist[1]
                     #Sort of meteors
                     indSort = finalHeights.argsort()
                     finalMeteor2 = finalMeteor[indSort,:]
                     #    Calculating winds
                     ind1 = 0
                     ind2 = 0
                     for i in range(nInt):
                         nMet = nMeteorsPerI[i]
                         ind1 = ind2
                         ind2 = ind1 + nMet
                         meteorAux = finalMeteor2[ind1:ind2,:]
                         if meteorAux.shape[0] >= meteorThresh:
                             vel = meteorAux[:, 6]
                             zen = meteorAux[:, 4]*numpy.pi/180
                             azim = meteorAux[:, 3]*numpy.pi/180
                             n = numpy.cos(zen)
                     #         m = (1 - n**2)/(1 - numpy.tan(azim)**2)
                     #         l = m*numpy.tan(azim)
                             l = numpy.sin(zen)*numpy.sin(azim)
                             m = numpy.sin(zen)*numpy.cos(azim)
                             A = numpy.vstack((l, m)).transpose()
                             A1 = numpy.dot(numpy.linalg.inv( numpy.dot(A.transpose(),A) ),A.transpose())
                             windsAux = numpy.dot(A1, vel)
                             winds[0,i] = windsAux[0]
                             winds[1,i] = windsAux[1]
                     return winds, heightPerI[:-1]
                 def techniqueNSM_SA(self, **kwargs):
                     metArray = kwargs['metArray']
                     heightList = kwargs['heightList']
                     timeList = kwargs['timeList']
                     rx_location = kwargs['rx_location']
                     groupList = kwargs['groupList']
                     azimuth = kwargs['azimuth']
                     dfactor = kwargs['dfactor']
                     k = kwargs['k']
                     azimuth1, dist = self.__calculateAzimuth1(rx_location, groupList, azimuth)
                     d = dist*dfactor
                     #Phase calculation
                     metArray1 = self.__getPhaseSlope(metArray, heightList, timeList)
                     metArray1[:,-2] = metArray1[:,-2]*metArray1[:,2]*1000/(k*d[metArray1[:,1].astype(int)]) #angles into velocities
                     velEst = numpy.zeros((heightList.size,2))*numpy.nan
                     azimuth1 = azimuth1*numpy.pi/180
                     for i in range(heightList.size):
                         h = heightList[i]
                         indH = numpy.where((metArray1[:,2] == h)&(numpy.abs(metArray1[:,-2]) < 100))[0]
                         metHeight = metArray1[indH,:]
                         if metHeight.shape[0] >= 2:
                             velAux = numpy.asmatrix(metHeight[:,-2]).T    #Radial Velocities
                             iazim = metHeight[:,1].astype(int)
                             azimAux = numpy.asmatrix(azimuth1[iazim]).T    #Azimuths
                             A = numpy.hstack((numpy.cos(azimAux),numpy.sin(azimAux)))
                             A = numpy.asmatrix(A)
                             A1 = numpy.linalg.pinv(A.transpose()*A)*A.transpose()
                             velHor = numpy.dot(A1,velAux)
                             velEst[i,:] = numpy.squeeze(velHor)
                     return velEst
                 def __getPhaseSlope(self, metArray, heightList, timeList):
                     meteorList = []
                     #utctime sec1 height SNR velRad ph0 ph1 ph2 coh0 coh1 coh2
                     #Putting back together the meteor matrix
                     utctime = metArray[:,0]
                     uniqueTime = numpy.unique(utctime)
                     phaseDerThresh = 0.5
                     ippSeconds = timeList[1] - timeList[0]
                     sec = numpy.where(timeList>1)[0][0]
                     nPairs = metArray.shape[1] - 6
                     nHeights = len(heightList)
                     for t in uniqueTime:
                         metArray1 = metArray[utctime==t,:]
             #         phaseDerThresh = numpy.pi/4 #reducir Phase thresh
                         tmet = metArray1[:,1].astype(int)
                         hmet = metArray1[:,2].astype(int)
                         metPhase = numpy.zeros((nPairs, heightList.size, timeList.size - 1))
                         metPhase[:,:] = numpy.nan
                         metPhase[:,hmet,tmet] = metArray1[:,6:].T
                         #Delete short trails
                         metBool = ~numpy.isnan(metPhase[0,:,:])
                         heightVect = numpy.sum(metBool, axis = 1)
                         metBool[heightVect<sec,:] = False
                         metPhase[:,heightVect<sec,:] = numpy.nan
                         #Derivative
                         metDer = numpy.abs(metPhase[:,:,1:] - metPhase[:,:,:-1])
                         phDerAux = numpy.dstack((numpy.full((nPairs,nHeights,1), False, dtype=bool),metDer > phaseDerThresh))
                         metPhase[phDerAux] = numpy.nan
                         #--------------------------METEOR DETECTION    -----------------------------------------
                         indMet = numpy.where(numpy.any(metBool,axis=1))[0]
                         for p in numpy.arange(nPairs):
                             phase = metPhase[p,:,:]
                             phDer = metDer[p,:,:]
                             for h in indMet:
                                 height = heightList[h]
                                 phase1 = phase[h,:] #82
                                 phDer1 = phDer[h,:]
                                 phase1[~numpy.isnan(phase1)] = numpy.unwrap(phase1[~numpy.isnan(phase1)])   #Unwrap
                                 indValid = numpy.where(~numpy.isnan(phase1))[0]
                                 initMet = indValid[0]
                                 endMet = 0
                                 for i in range(len(indValid)-1):
                                     #Time difference
                                     inow = indValid[i]
                                     inext = indValid[i+1]
                                     idiff = inext - inow
                                     #Phase difference
                                     phDiff = numpy.abs(phase1[inext] - phase1[inow])
                                     if idiff>sec or phDiff>numpy.pi/4 or inext==indValid[-1]:   #End of Meteor
                                         sizeTrail = inow - initMet + 1
                                         if sizeTrail>3*sec:  #Too short meteors
                                             x = numpy.arange(initMet,inow+1)*ippSeconds
                                             y = phase1[initMet:inow+1]
                                             ynnan = ~numpy.isnan(y)
                                             x = x[ynnan]
                                             y = y[ynnan]
                                             slope, intercept, r_value, p_value, std_err = stats.linregress(x,y)
                                             ylin = x*slope + intercept
                                             rsq = r_value**2
                                             if rsq > 0.5:
                                                 vel = slope#*height*1000/(k*d)
                                                 estAux = numpy.array([utctime,p,height, vel, rsq])
                                                 meteorList.append(estAux)
                                         initMet = inext
                     metArray2 = numpy.array(meteorList)
                     return metArray2
                 def __calculateAzimuth1(self, rx_location, pairslist, azimuth0):
                     azimuth1 = numpy.zeros(len(pairslist))
                     dist = numpy.zeros(len(pairslist))
                     for i in range(len(rx_location)):
                         ch0 = pairslist[i][0]
                         ch1 = pairslist[i][1]
                         diffX = rx_location[ch0][0] - rx_location[ch1][0]
                         diffY = rx_location[ch0][1] - rx_location[ch1][1]
                         azimuth1[i] = numpy.arctan2(diffY,diffX)*180/numpy.pi
                         dist[i] = numpy.sqrt(diffX**2 + diffY**2)
                     azimuth1 -= azimuth0
                     return azimuth1, dist
                 def techniqueNSM_DBS(self, **kwargs):
                     metArray = kwargs['metArray']
                     heightList = kwargs['heightList']
                     timeList = kwargs['timeList']
                     azimuth = kwargs['azimuth']
                     theta_x = numpy.array(kwargs['theta_x'])
                     theta_y = numpy.array(kwargs['theta_y'])
                     utctime = metArray[:,0]
                     cmet = metArray[:,1].astype(int)
                     hmet = metArray[:,3].astype(int)
                     SNRmet = metArray[:,4]
                     vmet = metArray[:,5]
                     spcmet = metArray[:,6]
                     nChan = numpy.max(cmet) + 1
                     nHeights = len(heightList)
                     azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw = self.__calculateAngles(theta_x, theta_y, azimuth)
                     hmet = heightList[hmet]
                     h1met = hmet*numpy.cos(zenith_arr[cmet])      #Corrected heights
                     velEst = numpy.zeros((heightList.size,2))*numpy.nan
                     for i in range(nHeights - 1):
                         hmin = heightList[i]
                         hmax = heightList[i + 1]
                         thisH = (h1met>=hmin) & (h1met<hmax) & (cmet!=2) & (SNRmet>8) & (vmet<50) & (spcmet<10)
                         indthisH = numpy.where(thisH)
                         if numpy.size(indthisH) > 3:
                             vel_aux = vmet[thisH]
                             chan_aux = cmet[thisH]
                             cosu_aux = dir_cosu[chan_aux]
                             cosv_aux = dir_cosv[chan_aux]
                             cosw_aux = dir_cosw[chan_aux]
                             nch = numpy.size(numpy.unique(chan_aux))
                             if  nch > 1:
                                 A = self.__calculateMatA(cosu_aux, cosv_aux, cosw_aux, True)
                                 velEst[i,:] = numpy.dot(A,vel_aux)
                     return velEst
                 def run(self, dataOut, technique, nHours=1, hmin=70, hmax=110, **kwargs):
                     param = dataOut.data_param
                     if dataOut.abscissaList != None:
                         absc = dataOut.abscissaList[:-1]
                     # noise = dataOut.noise
                     heightList = dataOut.heightList
                     SNR = dataOut.data_SNR
                     if technique == 'DBS':
                         kwargs['velRadial'] = param[:,1,:] #Radial velocity
                         kwargs['heightList'] = heightList
                         kwargs['SNR'] = SNR
                         dataOut.data_output, dataOut.heightList, dataOut.data_SNR = self.techniqueDBS(kwargs) #DBS Function
                         dataOut.utctimeInit = dataOut.utctime
                         dataOut.outputInterval = dataOut.paramInterval
                     elif technique == 'SA':
                         #Parameters
             #             position_x = kwargs['positionX']
             #             position_y = kwargs['positionY']
             #             azimuth = kwargs['azimuth']
             #
             #             if kwargs.has_key('crosspairsList'):
             #                 pairs = kwargs['crosspairsList']
             #             else:
             #                 pairs = None
             #
             #             if kwargs.has_key('correctFactor'):
             #                 correctFactor = kwargs['correctFactor']
             #             else:
             #                 correctFactor = 1
             #             tau = dataOut.data_param
             #             _lambda = dataOut.C/dataOut.frequency
             #             pairsList = dataOut.groupList
             #             nChannels = dataOut.nChannels
                         kwargs['groupList'] = dataOut.groupList
                         kwargs['tau'] = dataOut.data_param
                         kwargs['_lambda'] = dataOut.C/dataOut.frequency
             #             dataOut.data_output = self.techniqueSA(pairs, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, absc, correctFactor)
                         dataOut.data_output = self.techniqueSA(kwargs)
                         dataOut.utctimeInit = dataOut.utctime
                         dataOut.outputInterval = dataOut.timeInterval
                     elif technique == 'Meteors':
                         dataOut.flagNoData = True
                         self.__dataReady = False
                         if 'nHours' in kwargs:
                             nHours = kwargs['nHours']
                         else:
                             nHours = 1
                         if 'meteorsPerBin' in kwargs:
                             meteorThresh = kwargs['meteorsPerBin']
                         else:
                             meteorThresh = 6
                         if 'hmin' in kwargs:
                             hmin = kwargs['hmin']
                         else:   hmin = 70
                         if 'hmax' in kwargs:
                             hmax = kwargs['hmax']
                         else:   hmax = 110
                         dataOut.outputInterval = nHours*3600
                         if self.__isConfig == False:
             #                 self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
                             #Get Initial LTC time
                             self.__initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
                             self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
                             self.__isConfig = True
                         if self.__buffer is None:
                             self.__buffer = dataOut.data_param
                             self.__firstdata = copy.copy(dataOut)
                         else:
                             self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
                         self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
                         if self.__dataReady:
                             dataOut.utctimeInit = self.__initime
                             self.__initime += dataOut.outputInterval #to erase time offset
                             dataOut.data_output, dataOut.heightList = self.techniqueMeteors(self.__buffer, meteorThresh, hmin, hmax)
                             dataOut.flagNoData = False
                             self.__buffer = None
                     elif technique == 'Meteors1':
                         dataOut.flagNoData = True
                         self.__dataReady = False
                         if 'nMins' in kwargs:
                             nMins = kwargs['nMins']
                         else: nMins = 20
                         if 'rx_location' in kwargs:
                             rx_location = kwargs['rx_location']
                         else: rx_location = [(0,1),(1,1),(1,0)]
                         if 'azimuth' in kwargs:
                             azimuth = kwargs['azimuth']
                         else: azimuth = 51.06
                         if 'dfactor' in kwargs:
                             dfactor = kwargs['dfactor']
                         if 'mode' in kwargs:
                             mode = kwargs['mode']
                         if 'theta_x' in kwargs:
                             theta_x = kwargs['theta_x']
                         if 'theta_y' in kwargs:
                             theta_y = kwargs['theta_y']
                         else: mode = 'SA'
                         #Borrar luego esto
                         if dataOut.groupList is None:
                             dataOut.groupList = [(0,1),(0,2),(1,2)]
                         groupList = dataOut.groupList
                         C = 3e8
                         freq = 50e6
                         lamb = C/freq
                         k = 2*numpy.pi/lamb
                         timeList = dataOut.abscissaList
                         heightList = dataOut.heightList
                         if self.__isConfig == False:
                             dataOut.outputInterval = nMins*60
             #                 self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
                             #Get Initial LTC time
                             initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
                             minuteAux = initime.minute
                             minuteNew = int(numpy.floor(minuteAux/nMins)*nMins)
                             self.__initime = (initime.replace(minute = minuteNew, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
                             self.__isConfig = True
                         if self.__buffer is None:
                             self.__buffer = dataOut.data_param
                             self.__firstdata = copy.copy(dataOut)
                         else:
                             self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
                         self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
                         if self.__dataReady:
                             dataOut.utctimeInit = self.__initime
                             self.__initime += dataOut.outputInterval #to erase time offset
                             metArray = self.__buffer
                             if mode == 'SA':
                                 dataOut.data_output = self.techniqueNSM_SA(rx_location=rx_location, groupList=groupList, azimuth=azimuth, dfactor=dfactor, k=k,metArray=metArray, heightList=heightList,timeList=timeList)
                             elif mode == 'DBS':
                                 dataOut.data_output = self.techniqueNSM_DBS(metArray=metArray,heightList=heightList,timeList=timeList, azimuth=azimuth, theta_x=theta_x, theta_y=theta_y)
                             dataOut.data_output = dataOut.data_output.T
                             dataOut.flagNoData = False
                             self.__buffer = None
                     return
             class EWDriftsEstimation(Operation):
                 def __init__(self):
                     Operation.__init__(self)
                 def __correctValues(self, heiRang, phi, velRadial, SNR):
                     listPhi = phi.tolist()
                     maxid = listPhi.index(max(listPhi))
                     minid = listPhi.index(min(listPhi))
                     rango = list(range(len(phi)))
                #     rango = numpy.delete(rango,maxid)
                     heiRang1 = heiRang*math.cos(phi[maxid])
                     heiRangAux = heiRang*math.cos(phi[minid])
                     indOut = (heiRang1 < heiRangAux[0]).nonzero()
                     heiRang1 = numpy.delete(heiRang1,indOut)
                     velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
                     SNR1 = numpy.zeros([len(phi),len(heiRang1)])
                     for i in rango:
                         x = heiRang*math.cos(phi[i])
                         y1 = velRadial[i,:]
                         f1 = interpolate.interp1d(x,y1,kind = 'cubic')
                         x1 = heiRang1
                         y11 = f1(x1)
                         y2 = SNR[i,:]
                         f2 = interpolate.interp1d(x,y2,kind = 'cubic')
                         y21 = f2(x1)
                         velRadial1[i,:] = y11
                         SNR1[i,:] = y21
                     return heiRang1, velRadial1, SNR1
                 def run(self, dataOut, zenith, zenithCorrection):
                     heiRang = dataOut.heightList
                     velRadial = dataOut.data_param[:,3,:]
                     SNR = dataOut.data_SNR
                     zenith = numpy.array(zenith)
                     zenith -= zenithCorrection
                     zenith *= numpy.pi/180
                     heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, numpy.abs(zenith), velRadial, SNR)
                     alp = zenith[0]
                     bet = zenith[1]
                     w_w = velRadial1[0,:]
                     w_e = velRadial1[1,:]
                     w = (w_w*numpy.sin(bet) - w_e*numpy.sin(alp))/(numpy.cos(alp)*numpy.sin(bet) - numpy.cos(bet)*numpy.sin(alp))
                     u = (w_w*numpy.cos(bet) - w_e*numpy.cos(alp))/(numpy.sin(alp)*numpy.cos(bet) - numpy.sin(bet)*numpy.cos(alp))
                     winds = numpy.vstack((u,w))
                     dataOut.heightList = heiRang1
                     dataOut.data_output = winds
                     dataOut.data_SNR = SNR1
                     dataOut.utctimeInit = dataOut.utctime
                     dataOut.outputInterval = dataOut.timeInterval
                     return
             #---------------    Non Specular Meteor    ----------------
             class NonSpecularMeteorDetection(Operation):
                 def run(self, dataOut, mode, SNRthresh=8, phaseDerThresh=0.5, cohThresh=0.8, allData = False):
                     data_acf = dataOut.data_pre[0]
                     data_ccf = dataOut.data_pre[1]
                     pairsList = dataOut.groupList[1]
                     lamb = dataOut.C/dataOut.frequency
                     tSamp = dataOut.ippSeconds*dataOut.nCohInt
                     paramInterval = dataOut.paramInterval
                     nChannels = data_acf.shape[0]
                     nLags = data_acf.shape[1]
                     nProfiles = data_acf.shape[2]
                     nHeights = dataOut.nHeights
                     nCohInt = dataOut.nCohInt
                     sec = numpy.round(nProfiles/dataOut.paramInterval)
                     heightList = dataOut.heightList
                     ippSeconds = dataOut.ippSeconds*dataOut.nCohInt*dataOut.nAvg
                     utctime = dataOut.utctime
                     dataOut.abscissaList = numpy.arange(0,paramInterval+ippSeconds,ippSeconds)
                     #------------------------    SNR    --------------------------------------
                     power = data_acf[:,0,:,:].real
                     noise = numpy.zeros(nChannels)
                     SNR = numpy.zeros(power.shape)
                     for i in range(nChannels):
                         noise[i] = hildebrand_sekhon(power[i,:], nCohInt)
                         SNR[i] = (power[i]-noise[i])/noise[i]
                     SNRm = numpy.nanmean(SNR, axis = 0)
                     SNRdB = 10*numpy.log10(SNR)
                     if mode == 'SA':
                         dataOut.groupList = dataOut.groupList[1]
                         nPairs = data_ccf.shape[0]
                         #----------------------    Coherence and Phase   --------------------------
                         phase = numpy.zeros(data_ccf[:,0,:,:].shape)
             #             phase1 = numpy.copy(phase)
                         coh1 = numpy.zeros(data_ccf[:,0,:,:].shape)
                         for p in range(nPairs):
                             ch0 = pairsList[p][0]
                             ch1 = pairsList[p][1]
                             ccf = data_ccf[p,0,:,:]/numpy.sqrt(data_acf[ch0,0,:,:]*data_acf[ch1,0,:,:])
                             phase[p,:,:] = ndimage.median_filter(numpy.angle(ccf), size = (5,1)) #median filter
             #                 phase1[p,:,:] = numpy.angle(ccf) #median filter
                             coh1[p,:,:] = ndimage.median_filter(numpy.abs(ccf), 5) #median filter
             #                 coh1[p,:,:] = numpy.abs(ccf) #median filter
                         coh = numpy.nanmax(coh1, axis = 0)
             #             struc = numpy.ones((5,1))
             #             coh = ndimage.morphology.grey_dilation(coh, size=(10,1))
                         #----------------------    Radial Velocity    ----------------------------
                         phaseAux = numpy.mean(numpy.angle(data_acf[:,1,:,:]), axis = 0)
                         velRad = phaseAux*lamb/(4*numpy.pi*tSamp)
                         if allData:
                             boolMetFin = ~numpy.isnan(SNRm)
             #                 coh[:-1,:] = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
                         else:
                             #------------------------    Meteor mask    ---------------------------------
             #                 #SNR mask
             #                 boolMet = (SNRdB>SNRthresh)#|(~numpy.isnan(SNRdB))
             #
             #                 #Erase small objects
             #                 boolMet1 = self.__erase_small(boolMet, 2*sec, 5)
             #
             #                 auxEEJ = numpy.sum(boolMet1,axis=0)
             #                 indOver = auxEEJ>nProfiles*0.8  #Use this later
             #                 indEEJ = numpy.where(indOver)[0]
             #                 indNEEJ = numpy.where(~indOver)[0]
             #
             #                 boolMetFin = boolMet1
             #
             #                 if indEEJ.size > 0:
             #                     boolMet1[:,indEEJ] = False  #Erase heights with EEJ
             #
             #                     boolMet2 = coh > cohThresh
             #                     boolMet2 = self.__erase_small(boolMet2, 2*sec,5)
             #
             #                     #Final Meteor mask
             #                     boolMetFin = boolMet1|boolMet2
                             #Coherence mask
                             boolMet1 = coh > 0.75
                             struc = numpy.ones((30,1))
                             boolMet1 = ndimage.morphology.binary_dilation(boolMet1, structure=struc)
                             #Derivative mask
                             derPhase = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
                             boolMet2 = derPhase < 0.2
             #                 boolMet2 = ndimage.morphology.binary_opening(boolMet2)
             #                 boolMet2 = ndimage.morphology.binary_closing(boolMet2, structure = numpy.ones((10,1)))
                             boolMet2 = ndimage.median_filter(boolMet2,size=5)
                             boolMet2 = numpy.vstack((boolMet2,numpy.full((1,nHeights), True, dtype=bool)))
             #                 #Final mask
             #                 boolMetFin = boolMet2
                             boolMetFin = boolMet1&boolMet2
             #                 boolMetFin = ndimage.morphology.binary_dilation(boolMetFin)
                         #Creating data_param
                         coordMet = numpy.where(boolMetFin)
                         tmet = coordMet[0]
                         hmet = coordMet[1]
                         data_param = numpy.zeros((tmet.size, 6 + nPairs))
                         data_param[:,0] = utctime
                         data_param[:,1] = tmet
                         data_param[:,2] = hmet
                         data_param[:,3] = SNRm[tmet,hmet]
                         data_param[:,4] = velRad[tmet,hmet]
                         data_param[:,5] = coh[tmet,hmet]
                         data_param[:,6:] = phase[:,tmet,hmet].T
                     elif mode == 'DBS':
                         dataOut.groupList = numpy.arange(nChannels)
                         #Radial Velocities
                         phase = numpy.angle(data_acf[:,1,:,:])
             #             phase = ndimage.median_filter(numpy.angle(data_acf[:,1,:,:]), size = (1,5,1))
                         velRad = phase*lamb/(4*numpy.pi*tSamp)
                         #Spectral width
             #             acf1 = ndimage.median_filter(numpy.abs(data_acf[:,1,:,:]), size = (1,5,1))
             #             acf2 = ndimage.median_filter(numpy.abs(data_acf[:,2,:,:]), size = (1,5,1))
                         acf1 = data_acf[:,1,:,:]
                         acf2 = data_acf[:,2,:,:]
                         spcWidth = (lamb/(2*numpy.sqrt(6)*numpy.pi*tSamp))*numpy.sqrt(numpy.log(acf1/acf2))
             #             velRad = ndimage.median_filter(velRad, size = (1,5,1))
                         if allData:
                             boolMetFin = ~numpy.isnan(SNRdB)
                         else:
                             #SNR
                             boolMet1 = (SNRdB>SNRthresh) #SNR mask
                             boolMet1 = ndimage.median_filter(boolMet1, size=(1,5,5))
                             #Radial velocity
                             boolMet2 = numpy.abs(velRad) < 20
                             boolMet2 = ndimage.median_filter(boolMet2, (1,5,5))
                             #Spectral Width
                             boolMet3 = spcWidth < 30
                             boolMet3 = ndimage.median_filter(boolMet3, (1,5,5))
             #                 boolMetFin = self.__erase_small(boolMet1, 10,5)
                             boolMetFin = boolMet1&boolMet2&boolMet3
                         #Creating data_param
                         coordMet = numpy.where(boolMetFin)
                         cmet = coordMet[0]
                         tmet = coordMet[1]
                         hmet = coordMet[2]
                         data_param = numpy.zeros((tmet.size, 7))
                         data_param[:,0] = utctime
                         data_param[:,1] = cmet
                         data_param[:,2] = tmet
                         data_param[:,3] = hmet
                         data_param[:,4] = SNR[cmet,tmet,hmet].T
                         data_param[:,5] = velRad[cmet,tmet,hmet].T
                         data_param[:,6] = spcWidth[cmet,tmet,hmet].T
             #         self.dataOut.data_param = data_int
                     if len(data_param) == 0:
                         dataOut.flagNoData = True
                     else:
                         dataOut.data_param = data_param
                 def __erase_small(self, binArray, threshX, threshY):
                     labarray, numfeat = ndimage.measurements.label(binArray)
                     binArray1 = numpy.copy(binArray)
                     for i in range(1,numfeat + 1):
                         auxBin = (labarray==i)
                         auxSize = auxBin.sum()
                         x,y = numpy.where(auxBin)
                         widthX = x.max() - x.min()
                         widthY = y.max() - y.min()
                         #width X: 3 seg -> 12.5*3
                         #width Y:
                         if (auxSize < 50) or (widthX < threshX) or (widthY < threshY):
                             binArray1[auxBin] = False
                     return binArray1
             #---------------    Specular Meteor    ----------------
             class SMDetection(Operation):
                 '''
                     Function DetectMeteors()
                         Project developed with paper:
                         HOLDSWORTH ET AL. 2004
                     Input:
                         self.dataOut.data_pre
                         centerReceiverIndex:      From the channels, which is the center receiver
                         hei_ref:                  Height reference for the Beacon signal extraction
                         tauindex:
                         predefinedPhaseShifts:    Predefined phase offset for the voltge signals
                         cohDetection:             Whether to user Coherent detection or not
                         cohDet_timeStep:          Coherent Detection calculation time step
                         cohDet_thresh:            Coherent Detection phase threshold to correct phases
                         noise_timeStep:           Noise calculation time step
                         noise_multiple:           Noise multiple to define signal threshold
                         multDet_timeLimit:        Multiple Detection Removal time limit in seconds
                         multDet_rangeLimit:       Multiple Detection Removal range limit in km
                         phaseThresh:              Maximum phase difference between receiver to be consider a meteor
                         SNRThresh:                Minimum SNR threshold of the meteor signal to be consider a meteor
                         hmin:                     Minimum Height of the meteor to use it in the further wind estimations
                         hmax:                     Maximum Height of the meteor to use it in the further wind estimations
                         azimuth:                  Azimuth angle correction
                     Affected:
                         self.dataOut.data_param
                     Rejection Criteria (Errors):
 : No error; analysis OK
 : SNR < SNR threshold
 : angle of arrival (AOA) ambiguously determined
 : AOA estimate not feasible
 : Large difference in AOAs obtained from different antenna baselines
 : echo at start or end of time series
 : echo less than 5 examples long; too short for analysis
 : echo rise exceeds 0.3s
 : echo decay time less than twice rise time
 : large power level before echo
 : large power level after echo
 : poor fit to amplitude for estimation of decay time
 : poor fit to CCF phase variation for estimation of radial drift velocity
 : height unresolvable echo: not valid height within 70 to 110 km
 : height ambiguous echo: more then one possible height within 70 to 110 km
 : radial drift velocity or projected horizontal velocity exceeds 200 m/s
 : oscilatory echo, indicating event most likely not an underdense echo
 : phase difference in meteor Reestimation
                     Data Storage:
                         Meteors for Wind Estimation   (8):
                         Utc Time   |    Range    Height
                         Azimuth    Zenith    errorCosDir
                         VelRad    errorVelRad
                         Phase0 Phase1 Phase2 Phase3
                         TypeError
                      '''
                 def run(self, dataOut, hei_ref = None, tauindex = 0,
                                   phaseOffsets = None,
                                   cohDetection = False, cohDet_timeStep = 1, cohDet_thresh = 25,
                                   noise_timeStep = 4, noise_multiple = 4,
                                   multDet_timeLimit = 1, multDet_rangeLimit = 3,
                                   phaseThresh = 20, SNRThresh = 5,
                                   hmin = 50, hmax=150, azimuth = 0,
                                   channelPositions = None) :
                     #Getting Pairslist
                     if channelPositions is None:
             #             channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)]   #T
                         channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)]   #Estrella
                     meteorOps = SMOperations()
                     pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
                     heiRang = dataOut.getHeiRange()
                     #Get Beacon signal - No Beacon signal anymore
             #         newheis = numpy.where(self.dataOut.heightList>self.dataOut.radarControllerHeaderObj.Taus[tauindex])
             #
             #         if hei_ref != None:
             #             newheis = numpy.where(self.dataOut.heightList>hei_ref)
             #
                     #****************REMOVING HARDWARE PHASE DIFFERENCES***************
                     # see if the user put in pre defined phase shifts
                     voltsPShift = dataOut.data_pre.copy()
             #         if predefinedPhaseShifts != None:
             #             hardwarePhaseShifts = numpy.array(predefinedPhaseShifts)*numpy.pi/180
             #
             # #         elif beaconPhaseShifts:
             # #             #get hardware phase shifts using beacon signal
             # #             hardwarePhaseShifts = self.__getHardwarePhaseDiff(self.dataOut.data_pre, pairslist, newheis, 10)
             # #             hardwarePhaseShifts = numpy.insert(hardwarePhaseShifts,centerReceiverIndex,0)
             #
             #         else:
             #             hardwarePhaseShifts = numpy.zeros(5)
             #
             #         voltsPShift = numpy.zeros((self.dataOut.data_pre.shape[0],self.dataOut.data_pre.shape[1],self.dataOut.data_pre.shape[2]), dtype = 'complex')
             #         for i in range(self.dataOut.data_pre.shape[0]):
             #             voltsPShift[i,:,:] = self.__shiftPhase(self.dataOut.data_pre[i,:,:], hardwarePhaseShifts[i])
                     #******************END OF REMOVING HARDWARE PHASE DIFFERENCES*********
                     #Remove DC
                     voltsDC = numpy.mean(voltsPShift,1)
                     voltsDC = numpy.mean(voltsDC,1)
                     for i in range(voltsDC.shape[0]):
                         voltsPShift[i] = voltsPShift[i] - voltsDC[i]
                     #Don't considerate last heights, theyre used to calculate Hardware Phase Shift
             #         voltsPShift = voltsPShift[:,:,:newheis[0][0]]
                     #************ FIND POWER OF DATA W/COH OR NON COH DETECTION (3.4) **********
                     #Coherent Detection
                     if cohDetection:
                         #use coherent detection to get the net power
                         cohDet_thresh = cohDet_thresh*numpy.pi/180
                         voltsPShift = self.__coherentDetection(voltsPShift, cohDet_timeStep, dataOut.timeInterval, pairslist0, cohDet_thresh)
                     #Non-coherent detection!
                     powerNet = numpy.nansum(numpy.abs(voltsPShift[:,:,:])**2,0)
                     #********** END OF COH/NON-COH POWER CALCULATION**********************
                     #********** FIND THE NOISE LEVEL AND POSSIBLE METEORS ****************
                     #Get noise
                     noise, noise1 = self.__getNoise(powerNet, noise_timeStep, dataOut.timeInterval)
             #         noise = self.getNoise1(powerNet, noise_timeStep, self.dataOut.timeInterval)
                     #Get signal threshold
                     signalThresh = noise_multiple*noise
                     #Meteor echoes detection
                     listMeteors = self.__findMeteors(powerNet, signalThresh)
                     #******* END OF NOISE LEVEL AND POSSIBLE METEORS CACULATION **********
                     #************** REMOVE MULTIPLE DETECTIONS (3.5) ***************************
                     #Parameters
                     heiRange = dataOut.getHeiRange()
                     rangeInterval = heiRange[1] - heiRange[0]
                     rangeLimit = multDet_rangeLimit/rangeInterval
                     timeLimit = multDet_timeLimit/dataOut.timeInterval
                     #Multiple detection removals
                     listMeteors1 = self.__removeMultipleDetections(listMeteors, rangeLimit, timeLimit)
                     #************ END OF REMOVE MULTIPLE DETECTIONS **********************
                     #*********************     METEOR REESTIMATION  (3.7, 3.8, 3.9, 3.10)   ********************
                     #Parameters
                     phaseThresh = phaseThresh*numpy.pi/180
                     thresh = [phaseThresh, noise_multiple, SNRThresh]
                     #Meteor reestimation  (Errors N 1, 6, 12, 17)
                     listMeteors2, listMeteorsPower, listMeteorsVolts = self.__meteorReestimation(listMeteors1, voltsPShift, pairslist0, thresh, noise, dataOut.timeInterval, dataOut.frequency)
             #         listMeteors2, listMeteorsPower, listMeteorsVolts = self.meteorReestimation3(listMeteors2, listMeteorsPower, listMeteorsVolts, voltsPShift, pairslist, thresh, noise)
                     #Estimation of decay times (Errors N 7, 8, 11)
                     listMeteors3 = self.__estimateDecayTime(listMeteors2, listMeteorsPower, dataOut.timeInterval, dataOut.frequency)
                     #*******************     END OF METEOR REESTIMATION    *******************
                     #*********************    METEOR PARAMETERS CALCULATION (3.11, 3.12, 3.13)    **************************
                     #Calculating Radial Velocity (Error N 15)
                     radialStdThresh = 10
                     listMeteors4 = self.__getRadialVelocity(listMeteors3, listMeteorsVolts, radialStdThresh, pairslist0, dataOut.timeInterval)
                     if len(listMeteors4) > 0:
                         #Setting New Array
                         date = dataOut.utctime
                         arrayParameters = self.__setNewArrays(listMeteors4, date, heiRang)
                         #Correcting phase offset
                         if phaseOffsets != None:
                             phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
                             arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
                         #Second Pairslist
                         pairsList = []
                         pairx = (0,1)
                         pairy = (2,3)
                         pairsList.append(pairx)
                         pairsList.append(pairy)
                         jph = numpy.array([0,0,0,0])
                         h = (hmin,hmax)
                         arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
             #             #Calculate AOA (Error N 3, 4)
             #             #JONES ET AL. 1998
             #             error = arrayParameters[:,-1]
             #             AOAthresh = numpy.pi/8
             #             phases = -arrayParameters[:,9:13]
             #             arrayParameters[:,4:7], arrayParameters[:,-1] = meteorOps.getAOA(phases, pairsList, error, AOAthresh, azimuth)
             #
             #             #Calculate Heights (Error N 13 and 14)
             #             error = arrayParameters[:,-1]
             #             Ranges = arrayParameters[:,2]
             #             zenith = arrayParameters[:,5]
             #             arrayParameters[:,3], arrayParameters[:,-1] = meteorOps.getHeights(Ranges, zenith, error, hmin, hmax)
             #             error = arrayParameters[:,-1]
                     #*********************    END OF PARAMETERS CALCULATION    **************************
                     #***************************+     PASS DATA TO NEXT STEP    **********************
             #             arrayFinal = arrayParameters.reshape((1,arrayParameters.shape[0],arrayParameters.shape[1]))
                         dataOut.data_param = arrayParameters
                         if arrayParameters is None:
                             dataOut.flagNoData = True
                     else:
                         dataOut.flagNoData = True
                     return
                 def __getHardwarePhaseDiff(self, voltage0, pairslist, newheis, n):
                     minIndex = min(newheis[0])
                     maxIndex = max(newheis[0])
                     voltage = voltage0[:,:,minIndex:maxIndex+1]
                     nLength = voltage.shape[1]/n
                     nMin = 0
                     nMax = 0
                     phaseOffset = numpy.zeros((len(pairslist),n))
                     for i in range(n):
                         nMax += nLength
                         phaseCCF = -numpy.angle(self.__calculateCCF(voltage[:,nMin:nMax,:], pairslist, [0]))
                         phaseCCF = numpy.mean(phaseCCF, axis = 2)
                         phaseOffset[:,i] = phaseCCF.transpose()
                         nMin = nMax
             #         phaseDiff, phaseArrival = self.estimatePhaseDifference(voltage, pairslist)
                     #Remove Outliers
                     factor = 2
                     wt = phaseOffset - signal.medfilt(phaseOffset,(1,5))
                     dw = numpy.std(wt,axis = 1)
                     dw = dw.reshape((dw.size,1))
                     ind = numpy.where(numpy.logical_or(wt>dw*factor,wt<-dw*factor))
                     phaseOffset[ind] = numpy.nan
                     phaseOffset = stats.nanmean(phaseOffset, axis=1)
                     return phaseOffset
                 def __shiftPhase(self, data, phaseShift):
                     #this will shift the phase of a complex number
                     dataShifted = numpy.abs(data) * numpy.exp((numpy.angle(data)+phaseShift)*1j)
                     return dataShifted
                 def __estimatePhaseDifference(self, array, pairslist):
                     nChannel = array.shape[0]
                     nHeights = array.shape[2]
                     numPairs = len(pairslist)
             #         phaseCCF = numpy.zeros((nChannel, 5, nHeights))
                     phaseCCF = numpy.angle(self.__calculateCCF(array, pairslist, [-2,-1,0,1,2]))
                     #Correct phases
                     derPhaseCCF = phaseCCF[:,1:,:] - phaseCCF[:,0:-1,:]
                     indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
                     if indDer[0].shape[0] > 0:
                         for i in range(indDer[0].shape[0]):
                             signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i],indDer[2][i]])
                             phaseCCF[indDer[0][i],indDer[1][i]+1:,:] += signo*2*numpy.pi
             #         for j in range(numSides):
             #             phaseCCFAux = self.calculateCCF(arrayCenter, arraySides[j,:,:], [-2,1,0,1,2])
             #             phaseCCF[j,:,:] = numpy.angle(phaseCCFAux)
             #
                     #Linear
                     phaseInt = numpy.zeros((numPairs,1))
                     angAllCCF = phaseCCF[:,[0,1,3,4],0]
                     for j in range(numPairs):
                         fit = stats.linregress([-2,-1,1,2],angAllCCF[j,:])
                         phaseInt[j] = fit[1]
                     #Phase Differences
                     phaseDiff = phaseInt - phaseCCF[:,2,:]
                     phaseArrival = phaseInt.reshape(phaseInt.size)
                     #Dealias
                     phaseArrival = numpy.angle(numpy.exp(1j*phaseArrival))
             #         indAlias = numpy.where(phaseArrival > numpy.pi)
             #         phaseArrival[indAlias] -= 2*numpy.pi
             #         indAlias = numpy.where(phaseArrival < -numpy.pi)
             #         phaseArrival[indAlias] += 2*numpy.pi
                     return phaseDiff, phaseArrival
                 def __coherentDetection(self, volts, timeSegment, timeInterval, pairslist, thresh):
                     #this function will run the coherent detection used in Holdworth et al. 2004 and return the net power
                     #find the phase shifts of each channel over 1 second intervals
                     #only look at ranges below the beacon signal
                     numProfPerBlock = numpy.ceil(timeSegment/timeInterval)
                     numBlocks = int(volts.shape[1]/numProfPerBlock)
                     numHeights = volts.shape[2]
                     nChannel = volts.shape[0]
                     voltsCohDet = volts.copy()
                     pairsarray = numpy.array(pairslist)
                     indSides = pairsarray[:,1]
             #         indSides = numpy.array(range(nChannel))
             #         indSides = numpy.delete(indSides, indCenter)
             #
             #         listCenter = numpy.array_split(volts[indCenter,:,:], numBlocks, 0)
                     listBlocks = numpy.array_split(volts, numBlocks, 1)
                     startInd = 0
                     endInd = 0
                     for i in range(numBlocks):
                         startInd = endInd
                         endInd = endInd + listBlocks[i].shape[1]
                         arrayBlock = listBlocks[i]
             #             arrayBlockCenter = listCenter[i]
                         #Estimate the Phase Difference
                         phaseDiff, aux = self.__estimatePhaseDifference(arrayBlock, pairslist)
                         #Phase Difference RMS
                         arrayPhaseRMS = numpy.abs(phaseDiff)
                         phaseRMSaux = numpy.sum(arrayPhaseRMS < thresh,0)
                         indPhase = numpy.where(phaseRMSaux==4)
                         #Shifting
                         if indPhase[0].shape[0] > 0:
                             for j in range(indSides.size):
                                 arrayBlock[indSides[j],:,indPhase] = self.__shiftPhase(arrayBlock[indSides[j],:,indPhase], phaseDiff[j,indPhase].transpose())
                             voltsCohDet[:,startInd:endInd,:] = arrayBlock
                     return voltsCohDet
                 def __calculateCCF(self, volts, pairslist ,laglist):
                     nHeights = volts.shape[2]
                     nPoints = volts.shape[1]
                     voltsCCF = numpy.zeros((len(pairslist), len(laglist), nHeights),dtype = 'complex')
                     for i in range(len(pairslist)):
                         volts1 = volts[pairslist[i][0]]
                         volts2 = volts[pairslist[i][1]]
                         for t in range(len(laglist)):
                             idxT = laglist[t]
                             if idxT >= 0:
                                 vStacked = numpy.vstack((volts2[idxT:,:],
                                                        numpy.zeros((idxT, nHeights),dtype='complex')))
                             else:
                                 vStacked = numpy.vstack((numpy.zeros((-idxT, nHeights),dtype='complex'),
                                                        volts2[:(nPoints + idxT),:]))
                             voltsCCF[i,t,:] = numpy.sum((numpy.conjugate(volts1)*vStacked),axis=0)
                             vStacked = None
                     return voltsCCF
                 def __getNoise(self, power, timeSegment, timeInterval):
                     numProfPerBlock = numpy.ceil(timeSegment/timeInterval)
                     numBlocks = int(power.shape[0]/numProfPerBlock)
                     numHeights = power.shape[1]
                     listPower = numpy.array_split(power, numBlocks, 0)
                     noise = numpy.zeros((power.shape[0], power.shape[1]))
                     noise1 = numpy.zeros((power.shape[0], power.shape[1]))
                     startInd = 0
                     endInd = 0
                     for i in range(numBlocks):             #split por canal
                         startInd = endInd
                         endInd = endInd + listPower[i].shape[0]
                         arrayBlock = listPower[i]
                         noiseAux = numpy.mean(arrayBlock, 0)
             #             noiseAux = numpy.median(noiseAux)
             #             noiseAux = numpy.mean(arrayBlock)
                         noise[startInd:endInd,:] = noise[startInd:endInd,:] + noiseAux
                         noiseAux1 = numpy.mean(arrayBlock)
                         noise1[startInd:endInd,:] = noise1[startInd:endInd,:] + noiseAux1
                     return noise, noise1
                 def __findMeteors(self, power, thresh):
                     nProf = power.shape[0]
                     nHeights = power.shape[1]
                     listMeteors = []
                     for i in range(nHeights):
                         powerAux = power[:,i]
                         threshAux = thresh[:,i]
                         indUPthresh = numpy.where(powerAux > threshAux)[0]
                         indDNthresh = numpy.where(powerAux <= threshAux)[0]
                         j = 0
                         while (j < indUPthresh.size - 2):
                             if (indUPthresh[j + 2] == indUPthresh[j] + 2):
                                 indDNAux = numpy.where(indDNthresh > indUPthresh[j])
                                 indDNthresh = indDNthresh[indDNAux]
                                 if (indDNthresh.size > 0):
                                     indEnd = indDNthresh[0] - 1
                                     indInit = indUPthresh[j]
                                     meteor = powerAux[indInit:indEnd + 1]
                                     indPeak = meteor.argmax() + indInit
                                     FLA = sum(numpy.conj(meteor)*numpy.hstack((meteor[1:],0)))
                                     listMeteors.append(numpy.array([i,indInit,indPeak,indEnd,FLA])) #CHEQUEAR!!!!!
                                     j = numpy.where(indUPthresh == indEnd)[0] + 1
                                 else: j+=1
                             else: j+=1
                     return listMeteors
                 def __removeMultipleDetections(self,listMeteors, rangeLimit, timeLimit):
                     arrayMeteors = numpy.asarray(listMeteors)
                     listMeteors1 = []
                     while arrayMeteors.shape[0] > 0:
                         FLAs = arrayMeteors[:,4]
                         maxFLA = FLAs.argmax()
                         listMeteors1.append(arrayMeteors[maxFLA,:])
                         MeteorInitTime = arrayMeteors[maxFLA,1]
                         MeteorEndTime = arrayMeteors[maxFLA,3]
                         MeteorHeight = arrayMeteors[maxFLA,0]
                         #Check neighborhood
                         maxHeightIndex = MeteorHeight + rangeLimit
                         minHeightIndex = MeteorHeight - rangeLimit
                         minTimeIndex = MeteorInitTime - timeLimit
                         maxTimeIndex = MeteorEndTime + timeLimit
                         #Check Heights
                         indHeight = numpy.logical_and(arrayMeteors[:,0] >= minHeightIndex, arrayMeteors[:,0] <= maxHeightIndex)
                         indTime = numpy.logical_and(arrayMeteors[:,3] >= minTimeIndex, arrayMeteors[:,1] <= maxTimeIndex)
                         indBoth = numpy.where(numpy.logical_and(indTime,indHeight))
                         arrayMeteors = numpy.delete(arrayMeteors, indBoth, axis = 0)
                     return listMeteors1
                 def __meteorReestimation(self, listMeteors, volts, pairslist, thresh, noise, timeInterval,frequency):
                     numHeights = volts.shape[2]
                     nChannel = volts.shape[0]
                     thresholdPhase = thresh[0]
                     thresholdNoise = thresh[1]
                     thresholdDB = float(thresh[2])
                     thresholdDB1 = 10**(thresholdDB/10)
                     pairsarray = numpy.array(pairslist)
                     indSides = pairsarray[:,1]
                     pairslist1 = list(pairslist)
                     pairslist1.append((0,1))
                     pairslist1.append((3,4))
                     listMeteors1 = []
                     listPowerSeries = []
                     listVoltageSeries = []
                     #volts has the war data
                     if frequency == 30e6:
                         timeLag = 45*10**-3
                     else:
                         timeLag = 15*10**-3
                     lag = numpy.ceil(timeLag/timeInterval)
                     for i in range(len(listMeteors)):
                         ######################   3.6 - 3.7 PARAMETERS REESTIMATION    #########################
                         meteorAux = numpy.zeros(16)
                         #Loading meteor Data (mHeight, mStart, mPeak, mEnd)
                         mHeight = listMeteors[i][0]
                         mStart = listMeteors[i][1]
                         mPeak = listMeteors[i][2]
                         mEnd = listMeteors[i][3]
                         #get the volt data between the start and end times of the meteor
                         meteorVolts = volts[:,mStart:mEnd+1,mHeight]
                         meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
                         #3.6. Phase Difference estimation
                         phaseDiff, aux = self.__estimatePhaseDifference(meteorVolts, pairslist)
                         #3.7. Phase difference removal & meteor start, peak and end times reestimated
                         #meteorVolts0.- all Channels, all Profiles
                         meteorVolts0 = volts[:,:,mHeight]
                         meteorThresh = noise[:,mHeight]*thresholdNoise
                         meteorNoise = noise[:,mHeight]
                         meteorVolts0[indSides,:] = self.__shiftPhase(meteorVolts0[indSides,:], phaseDiff) #Phase Shifting
                         powerNet0 = numpy.nansum(numpy.abs(meteorVolts0)**2, axis = 0)  #Power
                         #Times reestimation
                         mStart1 = numpy.where(powerNet0[:mPeak] < meteorThresh[:mPeak])[0]
                         if mStart1.size > 0:
                             mStart1 = mStart1[-1] + 1
                         else:
                             mStart1 = mPeak
                         mEnd1 = numpy.where(powerNet0[mPeak:] < meteorThresh[mPeak:])[0][0] + mPeak - 1
                         mEndDecayTime1 = numpy.where(powerNet0[mPeak:] < meteorNoise[mPeak:])[0]
                         if mEndDecayTime1.size == 0:
                             mEndDecayTime1 = powerNet0.size
                         else:
                             mEndDecayTime1 = mEndDecayTime1[0] + mPeak - 1
             #             mPeak1 = meteorVolts0[mStart1:mEnd1 + 1].argmax()
                         #meteorVolts1.- all Channels, from start to end
                         meteorVolts1 = meteorVolts0[:,mStart1:mEnd1 + 1]
                         meteorVolts2 = meteorVolts0[:,mPeak + lag:mEnd1 + 1]
                         if meteorVolts2.shape[1] == 0:
                             meteorVolts2 = meteorVolts0[:,mPeak:mEnd1 + 1]
                         meteorVolts1 = meteorVolts1.reshape(meteorVolts1.shape[0], meteorVolts1.shape[1], 1)
                         meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1], 1)
                         #####################    END PARAMETERS REESTIMATION    #########################
                         #####################   3.8 PHASE DIFFERENCE REESTIMATION  ########################
             #             if mEnd1 - mStart1 > 4:       #Error Number 6: echo less than 5 samples long; too short for analysis
                         if meteorVolts2.shape[1] > 0:
                             #Phase Difference re-estimation
                             phaseDiff1, phaseDiffint = self.__estimatePhaseDifference(meteorVolts2, pairslist1)   #Phase Difference Estimation
             #                 phaseDiff1, phaseDiffint = self.estimatePhaseDifference(meteorVolts2, pairslist)
                             meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1])
                             phaseDiff11 = numpy.reshape(phaseDiff1, (phaseDiff1.shape[0],1))
                             meteorVolts2[indSides,:] = self.__shiftPhase(meteorVolts2[indSides,:], phaseDiff11[0:4])     #Phase Shifting
                             #Phase Difference RMS
                             phaseRMS1 = numpy.sqrt(numpy.mean(numpy.square(phaseDiff1)))
                             powerNet1 = numpy.nansum(numpy.abs(meteorVolts1[:,:])**2,0)
                             #Data from Meteor
                             mPeak1 = powerNet1.argmax() + mStart1
                             mPeakPower1 = powerNet1.max()
                             noiseAux = sum(noise[mStart1:mEnd1 + 1,mHeight])
                             mSNR1 = (sum(powerNet1)-noiseAux)/noiseAux
                             Meteor1 = numpy.array([mHeight, mStart1, mPeak1, mEnd1, mPeakPower1, mSNR1, phaseRMS1])
                             Meteor1 = numpy.hstack((Meteor1,phaseDiffint))
                             PowerSeries  = powerNet0[mStart1:mEndDecayTime1 + 1]
                             #Vectorize
                             meteorAux[0:7] = [mHeight, mStart1, mPeak1, mEnd1, mPeakPower1, mSNR1, phaseRMS1]
                             meteorAux[7:11] = phaseDiffint[0:4]
                             #Rejection Criterions
                             if phaseRMS1 > thresholdPhase:  #Error Number 17: Phase variation
                                 meteorAux[-1] = 17
                             elif mSNR1 < thresholdDB1:      #Error Number 1: SNR < threshold dB
                                 meteorAux[-1] = 1
                         else:
                             meteorAux[0:4] = [mHeight, mStart, mPeak, mEnd]
                             meteorAux[-1] = 6 #Error Number 6: echo less than 5 samples long; too short for analysis
                             PowerSeries = 0
                         listMeteors1.append(meteorAux)
                         listPowerSeries.append(PowerSeries)
                         listVoltageSeries.append(meteorVolts1)
                     return listMeteors1, listPowerSeries, listVoltageSeries
                 def __estimateDecayTime(self, listMeteors, listPower, timeInterval, frequency):
                     threshError = 10
                     #Depending if it is 30 or 50 MHz
                     if frequency == 30e6:
                         timeLag = 45*10**-3
                     else:
                         timeLag = 15*10**-3
                     lag = numpy.ceil(timeLag/timeInterval)
                     listMeteors1 = []
                     for i in range(len(listMeteors)):
                         meteorPower = listPower[i]
                         meteorAux = listMeteors[i]
                         if meteorAux[-1] == 0:
                             try:
                                 indmax = meteorPower.argmax()
                                 indlag = indmax + lag
                                 y = meteorPower[indlag:]
                                 x = numpy.arange(0, y.size)*timeLag
                                 #first guess
                                 a = y[0]
                                 tau = timeLag
                                 #exponential fit
                                 popt, pcov = optimize.curve_fit(self.__exponential_function, x, y, p0 = [a, tau])
                                 y1 = self.__exponential_function(x, *popt)
                                 #error estimation
                                 error = sum((y - y1)**2)/(numpy.var(y)*(y.size - popt.size))
                                 decayTime = popt[1]
                                 riseTime = indmax*timeInterval
                                 meteorAux[11:13] = [decayTime, error]
                                 #Table items 7, 8 and 11
                                 if (riseTime > 0.3):            #Number 7: Echo rise exceeds 0.3s
                                     meteorAux[-1] = 7
                                 elif (decayTime < 2*riseTime) : #Number 8: Echo decay time less than than twice rise time
                                     meteorAux[-1] = 8
                                 if (error > threshError):       #Number 11: Poor fit to amplitude for estimation of decay time
                                     meteorAux[-1] = 11
                             except:
                                 meteorAux[-1] = 11
                         listMeteors1.append(meteorAux)
                     return listMeteors1
                 #Exponential Function
                 def __exponential_function(self, x, a, tau):
                     y = a*numpy.exp(-x/tau)
                     return y
                 def __getRadialVelocity(self, listMeteors, listVolts, radialStdThresh, pairslist,  timeInterval):
                     pairslist1 = list(pairslist)
                     pairslist1.append((0,1))
                     pairslist1.append((3,4))
                     numPairs = len(pairslist1)
                     #Time Lag
                     timeLag = 45*10**-3
                     c = 3e8
                     lag = numpy.ceil(timeLag/timeInterval)
                     freq = 30e6
                     listMeteors1 = []
                     for i in range(len(listMeteors)):
                         meteorAux = listMeteors[i]
                         if meteorAux[-1] == 0:
                             mStart = listMeteors[i][1]
                             mPeak = listMeteors[i][2]
                             mLag = mPeak - mStart + lag
                             #get the volt data between the start and end times of the meteor
                             meteorVolts = listVolts[i]
                             meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
                             #Get CCF
                             allCCFs = self.__calculateCCF(meteorVolts, pairslist1, [-2,-1,0,1,2])
                             #Method 2
                             slopes = numpy.zeros(numPairs)
                             time = numpy.array([-2,-1,1,2])*timeInterval
                             angAllCCF = numpy.angle(allCCFs[:,[0,1,3,4],0])
                             #Correct phases
                             derPhaseCCF = angAllCCF[:,1:] - angAllCCF[:,0:-1]
                             indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
                             if indDer[0].shape[0] > 0:
                                 for i in range(indDer[0].shape[0]):
                                     signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i]])
                                     angAllCCF[indDer[0][i],indDer[1][i]+1:] += signo*2*numpy.pi
             #                     fit = scipy.stats.linregress(numpy.array([-2,-1,1,2])*timeInterval, numpy.array([phaseLagN2s[i],phaseLagN1s[i],phaseLag1s[i],phaseLag2s[i]]))
                             for j in range(numPairs):
                                 fit = stats.linregress(time, angAllCCF[j,:])
                                 slopes[j] = fit[0]
                             #Remove Outlier
             #                 indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
             #                 slopes = numpy.delete(slopes,indOut)
             #                 indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
             #                 slopes = numpy.delete(slopes,indOut)
                             radialVelocity = -numpy.mean(slopes)*(0.25/numpy.pi)*(c/freq)
                             radialError = numpy.std(slopes)*(0.25/numpy.pi)*(c/freq)
                             meteorAux[-2] = radialError
                             meteorAux[-3] = radialVelocity
                             #Setting Error
                             #Number 15: Radial Drift velocity or projected horizontal velocity exceeds 200 m/s
                             if numpy.abs(radialVelocity) > 200:
                                 meteorAux[-1] = 15
                             #Number 12: Poor fit to CCF variation for estimation of radial drift velocity
                             elif radialError > radialStdThresh:
                                 meteorAux[-1] = 12
                         listMeteors1.append(meteorAux)
                     return listMeteors1
                 def __setNewArrays(self, listMeteors, date, heiRang):
                     #New arrays
                     arrayMeteors = numpy.array(listMeteors)
                     arrayParameters = numpy.zeros((len(listMeteors), 13))
                     #Date inclusion
             #         date = re.findall(r'\((.*?)\)', date)
             #         date = date[0].split(',')
             #         date = map(int, date)
             #
             #         if len(date)<6:
             #             date.append(0)
             #
             #         date = [date[0]*10000 + date[1]*100 + date[2], date[3]*10000 + date[4]*100 + date[5]]
             #         arrayDate = numpy.tile(date, (len(listMeteors), 1))
                     arrayDate = numpy.tile(date, (len(listMeteors)))
                     #Meteor array
             #         arrayMeteors[:,0] = heiRang[arrayMeteors[:,0].astype(int)]
             #         arrayMeteors = numpy.hstack((arrayDate, arrayMeteors))
                     #Parameters Array
                     arrayParameters[:,0] = arrayDate #Date
                     arrayParameters[:,1] = heiRang[arrayMeteors[:,0].astype(int)] #Range
                     arrayParameters[:,6:8] = arrayMeteors[:,-3:-1] #Radial velocity and its error
                     arrayParameters[:,8:12] = arrayMeteors[:,7:11]  #Phases
                     arrayParameters[:,-1] = arrayMeteors[:,-1]  #Error
                     return arrayParameters
             class CorrectSMPhases(Operation):
                 def run(self, dataOut, phaseOffsets, hmin = 50, hmax = 150, azimuth = 45, channelPositions = None):
                     arrayParameters = dataOut.data_param
                     pairsList = []
                     pairx = (0,1)
                     pairy = (2,3)
                     pairsList.append(pairx)
                     pairsList.append(pairy)
                     jph = numpy.zeros(4)
                     phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
                 #         arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
                     arrayParameters[:,8:12] = numpy.angle(numpy.exp(1j*(arrayParameters[:,8:12] + phaseOffsets)))
                     meteorOps = SMOperations()
                     if channelPositions is None:
                 #             channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)]   #T
                         channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)]   #Estrella
                     pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
                     h = (hmin,hmax)
                     arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
                     dataOut.data_param = arrayParameters
                     return
             class SMPhaseCalibration(Operation):
                 __buffer = None
                 __initime = None
                 __dataReady = False
                 __isConfig = False
                 def __checkTime(self, currentTime, initTime, paramInterval, outputInterval):
                     dataTime = currentTime + paramInterval
                     deltaTime = dataTime - initTime
                     if deltaTime >= outputInterval or deltaTime < 0:
                         return True
                     return False
                 def __getGammas(self, pairs, d, phases):
                     gammas = numpy.zeros(2)
                     for i in range(len(pairs)):
                         pairi = pairs[i]
                         phip3 = phases[:,pairi[0]]
                         d3 = d[pairi[0]]
                         phip2 = phases[:,pairi[1]]
                         d2 = d[pairi[1]]
                         #Calculating gamma
             #             jdcos = alp1/(k*d1)
             #             jgamma = numpy.angle(numpy.exp(1j*(d0*alp1/d1 - alp0)))
                         jgamma = -phip2*d3/d2 - phip3
                         jgamma = numpy.angle(numpy.exp(1j*jgamma))
             #             jgamma[jgamma>numpy.pi] -= 2*numpy.pi
             #             jgamma[jgamma<-numpy.pi] += 2*numpy.pi
                         #Revised distribution
                         jgammaArray = numpy.hstack((jgamma,jgamma+0.5*numpy.pi,jgamma-0.5*numpy.pi))
                         #Histogram
                         nBins = 64
                         rmin = -0.5*numpy.pi
                         rmax = 0.5*numpy.pi
                         phaseHisto = numpy.histogram(jgammaArray, bins=nBins, range=(rmin,rmax))
                         meteorsY = phaseHisto[0]
                         phasesX = phaseHisto[1][:-1]
                         width = phasesX[1] - phasesX[0]
                         phasesX += width/2
                         #Gaussian aproximation
                         bpeak = meteorsY.argmax()
                         peak = meteorsY.max()
                         jmin = bpeak - 5
                         jmax = bpeak + 5 + 1
                         if jmin<0:
                             jmin = 0
                             jmax = 6
                         elif jmax > meteorsY.size:
                             jmin = meteorsY.size - 6
                             jmax = meteorsY.size
                         x0 = numpy.array([peak,bpeak,50])
                         coeff = optimize.leastsq(self.__residualFunction, x0, args=(meteorsY[jmin:jmax], phasesX[jmin:jmax]))
                         #Gammas
                         gammas[i] = coeff[0][1]
                     return gammas
                 def __residualFunction(self, coeffs, y, t):
                     return y - self.__gauss_function(t, coeffs)
                 def __gauss_function(self, t, coeffs):
                     return coeffs[0]*numpy.exp(-0.5*((t - coeffs[1]) / coeffs[2])**2)
                 def __getPhases(self, azimuth, h, pairsList, d, gammas, meteorsArray):
                     meteorOps = SMOperations()
                     nchan = 4
                     pairx = pairsList[0] #x es 0
                     pairy = pairsList[1] #y es 1
                     center_xangle = 0
                     center_yangle = 0
                     range_angle = numpy.array([10*numpy.pi,numpy.pi,numpy.pi/2,numpy.pi/4])
                     ntimes = len(range_angle)
                     nstepsx = 20
                     nstepsy = 20
                     for iz in range(ntimes):
                         min_xangle = -range_angle[iz]/2 + center_xangle
                         max_xangle = range_angle[iz]/2 + center_xangle
                         min_yangle = -range_angle[iz]/2 + center_yangle
                         max_yangle = range_angle[iz]/2 + center_yangle
                         inc_x = (max_xangle-min_xangle)/nstepsx
                         inc_y = (max_yangle-min_yangle)/nstepsy
                         alpha_y = numpy.arange(nstepsy)*inc_y + min_yangle
                         alpha_x = numpy.arange(nstepsx)*inc_x + min_xangle
                         penalty = numpy.zeros((nstepsx,nstepsy))
                         jph_array = numpy.zeros((nchan,nstepsx,nstepsy))
                         jph = numpy.zeros(nchan)
                         # Iterations looking for the offset
                         for iy in range(int(nstepsy)):
                             for ix in range(int(nstepsx)):
                                 d3 = d[pairsList[1][0]]
                                 d2 = d[pairsList[1][1]]
                                 d5 = d[pairsList[0][0]]
                                 d4 = d[pairsList[0][1]]
                                 alp2 = alpha_y[iy]  #gamma 1
                                 alp4 = alpha_x[ix]  #gamma 0
                                 alp3 = -alp2*d3/d2 - gammas[1]
                                 alp5 = -alp4*d5/d4 - gammas[0]
             #                     jph[pairy[1]] = alpha_y[iy]
             #                     jph[pairy[0]] = -gammas[1] - alpha_y[iy]*d[pairy[1]]/d[pairy[0]]
             #                     jph[pairx[1]] = alpha_x[ix]
             #                     jph[pairx[0]] = -gammas[0] - alpha_x[ix]*d[pairx[1]]/d[pairx[0]]
                                 jph[pairsList[0][1]] = alp4
                                 jph[pairsList[0][0]] = alp5
                                 jph[pairsList[1][0]] = alp3
                                 jph[pairsList[1][1]] = alp2
                                 jph_array[:,ix,iy] = jph
             #                     d = [2.0,2.5,2.5,2.0]
                                 #falta chequear si va a leer bien  los meteoros
                                 meteorsArray1 = meteorOps.getMeteorParams(meteorsArray, azimuth, h, pairsList, d, jph)
                                 error = meteorsArray1[:,-1]
                                 ind1 = numpy.where(error==0)[0]
                                 penalty[ix,iy] = ind1.size
                         i,j = numpy.unravel_index(penalty.argmax(), penalty.shape)
                         phOffset = jph_array[:,i,j]
                         center_xangle = phOffset[pairx[1]]
                         center_yangle = phOffset[pairy[1]]
                     phOffset = numpy.angle(numpy.exp(1j*jph_array[:,i,j]))
                     phOffset = phOffset*180/numpy.pi
                     return phOffset
                 def run(self, dataOut, hmin, hmax, channelPositions=None, nHours = 1):
                     dataOut.flagNoData = True
                     self.__dataReady = False
                     dataOut.outputInterval = nHours*3600
                     if self.__isConfig == False:
             #                 self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
                         #Get Initial LTC time
                         self.__initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
                         self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
                         self.__isConfig = True
                     if self.__buffer is None:
                         self.__buffer = dataOut.data_param.copy()
                     else:
                         self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
                     self.__dataReady = self.__checkTime(dataOut.utctime, self.__initime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
                     if self.__dataReady:
                         dataOut.utctimeInit = self.__initime
                         self.__initime += dataOut.outputInterval #to erase time offset
                         freq = dataOut.frequency
                         c = dataOut.C #m/s
                         lamb = c/freq
                         k = 2*numpy.pi/lamb
                         azimuth = 0
                         h = (hmin, hmax)
             #             pairs = ((0,1),(2,3)) #Estrella
             #             pairs = ((1,0),(2,3)) #T
                         if channelPositions is None:
             #             channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)]   #T
                             channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)]   #Estrella
                         meteorOps = SMOperations()
                         pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
                         #Checking correct order of pairs
                         pairs = []
                         if distances[1] > distances[0]:
                             pairs.append((1,0))
                         else:
                             pairs.append((0,1))
                         if distances[3] > distances[2]:
                             pairs.append((3,2))
                         else:
                             pairs.append((2,3))
             #             distances1 = [-distances[0]*lamb, distances[1]*lamb, -distances[2]*lamb, distances[3]*lamb]
                         meteorsArray = self.__buffer
                         error = meteorsArray[:,-1]
                         boolError = (error==0)|(error==3)|(error==4)|(error==13)|(error==14)
                         ind1 = numpy.where(boolError)[0]
                         meteorsArray = meteorsArray[ind1,:]
                         meteorsArray[:,-1] = 0
                         phases = meteorsArray[:,8:12]
                         #Calculate Gammas
                         gammas = self.__getGammas(pairs, distances, phases)
             #             gammas = numpy.array([-21.70409463,45.76935864])*numpy.pi/180
                         #Calculate Phases
                         phasesOff = self.__getPhases(azimuth, h, pairs, distances, gammas, meteorsArray)
                         phasesOff = phasesOff.reshape((1,phasesOff.size))
                         dataOut.data_output = -phasesOff
                         dataOut.flagNoData = False
                         self.__buffer = None
                     return
             class SMOperations():
                 def __init__(self):
                     return
                 def getMeteorParams(self, arrayParameters0, azimuth, h, pairsList, distances, jph):
                     arrayParameters = arrayParameters0.copy()
                     hmin = h[0]
                     hmax = h[1]
                     #Calculate AOA (Error N 3, 4)
                     #JONES ET AL. 1998
                     AOAthresh = numpy.pi/8
                     error = arrayParameters[:,-1]
                     phases = -arrayParameters[:,8:12] + jph
             #         phases = numpy.unwrap(phases)
                     arrayParameters[:,3:6], arrayParameters[:,-1] = self.__getAOA(phases, pairsList, distances, error, AOAthresh, azimuth)
                     #Calculate Heights (Error N 13 and 14)
                     error = arrayParameters[:,-1]
                     Ranges = arrayParameters[:,1]
                     zenith = arrayParameters[:,4]
                     arrayParameters[:,2], arrayParameters[:,-1] = self.__getHeights(Ranges, zenith, error, hmin, hmax)
                     #----------------------- Get Final data    ------------------------------------
             #         error = arrayParameters[:,-1]
             #         ind1 = numpy.where(error==0)[0]
             #         arrayParameters = arrayParameters[ind1,:]
                     return arrayParameters
                 def __getAOA(self, phases, pairsList, directions, error, AOAthresh, azimuth):
                     arrayAOA = numpy.zeros((phases.shape[0],3))
                     cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList,directions)
                     arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
                     cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
                     arrayAOA[:,2] = cosDirError
                     azimuthAngle = arrayAOA[:,0]
                     zenithAngle = arrayAOA[:,1]
                     #Setting Error
                     indError = numpy.where(numpy.logical_or(error == 3, error == 4))[0]
                     error[indError] = 0
                     #Number 3: AOA not fesible
                     indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
                     error[indInvalid] = 3
                     #Number 4: Large difference in AOAs obtained from different antenna baselines
                     indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
                     error[indInvalid] = 4
                     return arrayAOA, error
                 def __getDirectionCosines(self, arrayPhase, pairsList, distances):
                     #Initializing some variables
                     ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
                     ang_aux = ang_aux.reshape(1,ang_aux.size)
                     cosdir = numpy.zeros((arrayPhase.shape[0],2))
                     cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
                     for i in range(2):
                         ph0 = arrayPhase[:,pairsList[i][0]]
                         ph1 = arrayPhase[:,pairsList[i][1]]
                         d0 = distances[pairsList[i][0]]
                         d1 = distances[pairsList[i][1]]
                         ph0_aux = ph0 + ph1
                         ph0_aux = numpy.angle(numpy.exp(1j*ph0_aux))
             #             ph0_aux[ph0_aux > numpy.pi] -= 2*numpy.pi
             #             ph0_aux[ph0_aux < -numpy.pi] += 2*numpy.pi
                         #First Estimation
                         cosdir0[:,i] = (ph0_aux)/(2*numpy.pi*(d0 - d1))
                         #Most-Accurate Second Estimation
                         phi1_aux =  ph0 - ph1
                         phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
                         #Direction Cosine 1
                         cosdir1 = (phi1_aux + ang_aux)/(2*numpy.pi*(d0 + d1))
                         #Searching the correct Direction Cosine
                         cosdir0_aux = cosdir0[:,i]
                         cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
                         #Minimum Distance
                         cosDiff = (cosdir1 - cosdir0_aux)**2
                         indcos = cosDiff.argmin(axis = 1)
                         #Saving Value obtained
                         cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
                     return cosdir0, cosdir
                 def __calculateAOA(self, cosdir, azimuth):
                     cosdirX = cosdir[:,0]
                     cosdirY = cosdir[:,1]
                     zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
                     azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth#0 deg north, 90 deg east
                     angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
                     return angles
                 def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
                     Ramb = 375  #Ramb = c/(2*PRF)
                     Re = 6371   #Earth Radius
                     heights = numpy.zeros(Ranges.shape)
                     R_aux = numpy.array([0,1,2])*Ramb
                     R_aux = R_aux.reshape(1,R_aux.size)
                     Ranges = Ranges.reshape(Ranges.size,1)
                     Ri = Ranges + R_aux
                     hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
                     #Check if there is a height between 70 and 110 km
                     h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
                     ind_h = numpy.where(h_bool == 1)[0]
                     hCorr = hi[ind_h, :]
                     ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
                     hCorr = hi[ind_hCorr][:len(ind_h)]
                     heights[ind_h] = hCorr
                     #Setting Error
                     #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
                     #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
                     indError = numpy.where(numpy.logical_or(error == 13, error == 14))[0]
                     error[indError] = 0
                     indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
                     error[indInvalid2] = 14
                     indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
                     error[indInvalid1] = 13
                     return heights, error
                 def getPhasePairs(self, channelPositions):
                     chanPos = numpy.array(channelPositions)
                     listOper = list(itertools.combinations(list(range(5)),2))
                     distances = numpy.zeros(4)
                     axisX = []
                     axisY = []
                     distX = numpy.zeros(3)
                     distY = numpy.zeros(3)
                     ix = 0
                     iy = 0
                     pairX = numpy.zeros((2,2))
                     pairY = numpy.zeros((2,2))
                     for i in range(len(listOper)):
                         pairi = listOper[i]
                         posDif = numpy.abs(chanPos[pairi[0],:] - chanPos[pairi[1],:])
                         if posDif[0] == 0:
                             axisY.append(pairi)
                             distY[iy] = posDif[1]
                             iy += 1
                         elif posDif[1] == 0:
                             axisX.append(pairi)
                             distX[ix] = posDif[0]
                             ix += 1
                     for i in range(2):
                         if i==0:
                             dist0 = distX
                             axis0 = axisX
                         else:
                             dist0 = distY
                             axis0 = axisY
                         side = numpy.argsort(dist0)[:-1]
                         axis0 = numpy.array(axis0)[side,:]
                         chanC = int(numpy.intersect1d(axis0[0,:], axis0[1,:])[0])
                         axis1 = numpy.unique(numpy.reshape(axis0,4))
                         side = axis1[axis1 != chanC]
                         diff1 = chanPos[chanC,i] - chanPos[side[0],i]
                         diff2 = chanPos[chanC,i] - chanPos[side[1],i]
                         if diff1<0:
                             chan2 = side[0]
                             d2 = numpy.abs(diff1)
                             chan1 = side[1]
                             d1 = numpy.abs(diff2)
                         else:
                             chan2 = side[1]
                             d2 = numpy.abs(diff2)
                             chan1 = side[0]
                             d1 = numpy.abs(diff1)
                         if i==0:
                             chanCX = chanC
                             chan1X = chan1
                             chan2X = chan2
                             distances[0:2] = numpy.array([d1,d2])
                         else:
                             chanCY = chanC
                             chan1Y = chan1
                             chan2Y = chan2
                             distances[2:4] = numpy.array([d1,d2])
             #         axisXsides = numpy.reshape(axisX[ix,:],4)
             #
             #         channelCentX = int(numpy.intersect1d(pairX[0,:], pairX[1,:])[0])
             #         channelCentY = int(numpy.intersect1d(pairY[0,:], pairY[1,:])[0])
             #
             #         ind25X = numpy.where(pairX[0,:] != channelCentX)[0][0]
             #         ind20X = numpy.where(pairX[1,:] != channelCentX)[0][0]
             #         channel25X = int(pairX[0,ind25X])
             #         channel20X = int(pairX[1,ind20X])
             #         ind25Y = numpy.where(pairY[0,:] != channelCentY)[0][0]
             #         ind20Y = numpy.where(pairY[1,:] != channelCentY)[0][0]
             #         channel25Y = int(pairY[0,ind25Y])
             #         channel20Y = int(pairY[1,ind20Y])
             #         pairslist = [(channelCentX, channel25X),(channelCentX, channel20X),(channelCentY,channel25Y),(channelCentY, channel20Y)]
                     pairslist = [(chanCX, chan1X),(chanCX, chan2X),(chanCY,chan1Y),(chanCY, chan2Y)]
                     return pairslist, distances
             #     def __getAOA(self, phases, pairsList, error, AOAthresh, azimuth):
             #
             #         arrayAOA = numpy.zeros((phases.shape[0],3))
             #         cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList)
             #
             #         arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
             #         cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
             #         arrayAOA[:,2] = cosDirError
             #
             #         azimuthAngle = arrayAOA[:,0]
             #         zenithAngle = arrayAOA[:,1]
             #
             #         #Setting Error
             #         #Number 3: AOA not fesible
             #         indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
             #         error[indInvalid] = 3
             #         #Number 4: Large difference in AOAs obtained from different antenna baselines
             #         indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
             #         error[indInvalid] = 4
             #         return arrayAOA, error
             #
             #     def __getDirectionCosines(self, arrayPhase, pairsList):
             #
             #         #Initializing some variables
             #         ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
             #         ang_aux = ang_aux.reshape(1,ang_aux.size)
             #
             #         cosdir = numpy.zeros((arrayPhase.shape[0],2))
             #         cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
             #
             #
             #         for i in range(2):
             #             #First Estimation
             #             phi0_aux = arrayPhase[:,pairsList[i][0]] + arrayPhase[:,pairsList[i][1]]
             #             #Dealias
             #             indcsi = numpy.where(phi0_aux > numpy.pi)
             #             phi0_aux[indcsi] -= 2*numpy.pi
             #             indcsi = numpy.where(phi0_aux < -numpy.pi)
             #             phi0_aux[indcsi] += 2*numpy.pi
             #             #Direction Cosine 0
             #             cosdir0[:,i] = -(phi0_aux)/(2*numpy.pi*0.5)
             #
             #             #Most-Accurate Second Estimation
             #             phi1_aux =  arrayPhase[:,pairsList[i][0]] - arrayPhase[:,pairsList[i][1]]
             #             phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
             #             #Direction Cosine 1
             #             cosdir1 = -(phi1_aux + ang_aux)/(2*numpy.pi*4.5)
             #
             #             #Searching the correct Direction Cosine
             #             cosdir0_aux = cosdir0[:,i]
             #             cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
             #             #Minimum Distance
             #             cosDiff = (cosdir1 - cosdir0_aux)**2
             #             indcos = cosDiff.argmin(axis = 1)
             #             #Saving Value obtained
             #             cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
             #
             #         return cosdir0, cosdir
             #
             #     def __calculateAOA(self, cosdir, azimuth):
             #         cosdirX = cosdir[:,0]
             #         cosdirY = cosdir[:,1]
             #
             #         zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
             #         azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth #0 deg north, 90 deg east
             #         angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
             #
             #         return angles
             #
             #     def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
             #
             #         Ramb = 375  #Ramb = c/(2*PRF)
             #         Re = 6371   #Earth Radius
             #         heights = numpy.zeros(Ranges.shape)
             #
             #         R_aux = numpy.array([0,1,2])*Ramb
             #         R_aux = R_aux.reshape(1,R_aux.size)
             #
             #         Ranges = Ranges.reshape(Ranges.size,1)
             #
             #         Ri = Ranges + R_aux
             #         hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
             #
             #         #Check if there is a height between 70 and 110 km
             #         h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
             #         ind_h = numpy.where(h_bool == 1)[0]
             #
             #         hCorr = hi[ind_h, :]
             #         ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
             #
             #         hCorr = hi[ind_hCorr]
             #         heights[ind_h] = hCorr
             #
             #         #Setting Error
             #         #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
             #         #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
             #
             #         indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
             #         error[indInvalid2] = 14
             #         indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
             #         error[indInvalid1] = 13
             #
             #         return heights, error
-  No newline at end of file

schainpy/model/proc/jroproc_spectra.py 0 0

NO CONTENT: modified file

schainpy/model/proc/jroproc_voltage.py 0 -1

             import sys
             import numpy
             from scipy import interpolate
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jrodata import Voltage
             from schainpy.utils import log
             from time import time
             @MPDecorator
             class VoltageProc(ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Voltage()
                     self.flip = 1
                     self.setupReq = False
                 def run(self):
                     if self.dataIn.type == 'AMISR':
                         self.__updateObjFromAmisrInput()
                     if self.dataIn.type == 'Voltage':
                         self.dataOut.copy(self.dataIn)
                 #         self.dataOut.copy(self.dataIn)
                 def __updateObjFromAmisrInput(self):
                     self.dataOut.timeZone = self.dataIn.timeZone
                     self.dataOut.dstFlag = self.dataIn.dstFlag
                     self.dataOut.errorCount = self.dataIn.errorCount
                     self.dataOut.useLocalTime = self.dataIn.useLocalTime
                     self.dataOut.flagNoData = self.dataIn.flagNoData
                     self.dataOut.data = self.dataIn.data
                     self.dataOut.utctime = self.dataIn.utctime
                     self.dataOut.channelList = self.dataIn.channelList
                     #self.dataOut.timeInterval = self.dataIn.timeInterval
                     self.dataOut.heightList = self.dataIn.heightList
                     self.dataOut.nProfiles = self.dataIn.nProfiles
                     self.dataOut.nCohInt = self.dataIn.nCohInt
                     self.dataOut.ippSeconds = self.dataIn.ippSeconds
                     self.dataOut.frequency = self.dataIn.frequency
                     self.dataOut.azimuth = self.dataIn.azimuth
                     self.dataOut.zenith = self.dataIn.zenith
                     self.dataOut.beam.codeList = self.dataIn.beam.codeList
                     self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
                     self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
                 #
                 #        pass#
                 #
                 #    def init(self):
                 #
                 #
                 #        if self.dataIn.type == 'AMISR':
                 #            self.__updateObjFromAmisrInput()
                 #
                 #        if self.dataIn.type == 'Voltage':
                 #            self.dataOut.copy(self.dataIn)
                 #        # No necesita copiar en cada init() los atributos de dataIn
                 #        # la copia deberia hacerse por cada nuevo bloque de datos
                 def selectChannels(self, channelList):
                     channelIndexList = []
                     for channel in channelList:
                         if channel not in self.dataOut.channelList:
                             raise ValueError("Channel %d is not in %s" %(channel, str(self.dataOut.channelList)))
                         index = self.dataOut.channelList.index(channel)
                         channelIndexList.append(index)
                     self.selectChannelsByIndex(channelIndexList)
                 def selectChannelsByIndex(self, channelIndexList):
                     """
                     Selecciona un bloque de datos en base a canales segun el channelIndexList
                     Input:
                         channelIndexList    :    lista sencilla de canales a seleccionar por ej. [2,3,7]
                     Affected:
                         self.dataOut.data
                         self.dataOut.channelIndexList
                         self.dataOut.nChannels
                         self.dataOut.m_ProcessingHeader.totalSpectra
                         self.dataOut.systemHeaderObj.numChannels
                         self.dataOut.m_ProcessingHeader.blockSize
                     Return:
                         None
                     """
                     for channelIndex in channelIndexList:
                         if channelIndex not in self.dataOut.channelIndexList:
                             print(channelIndexList)
                             raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
                     if self.dataOut.flagDataAsBlock:
                         """
                         Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         data = self.dataOut.data[channelIndexList,:,:]
                     else:
                         data = self.dataOut.data[channelIndexList,:]
                     self.dataOut.data = data
                     # self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
                     self.dataOut.channelList = range(len(channelIndexList))
                     return 1
                 def selectHeights(self, minHei=None, maxHei=None):
                     """
                     Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
                     minHei <= height <= maxHei
                     Input:
                         minHei    :    valor minimo de altura a considerar
                         maxHei    :    valor maximo de altura a considerar
                     Affected:
                         Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if minHei == None:
                         minHei = self.dataOut.heightList[0]
                     if maxHei == None:
                         maxHei = self.dataOut.heightList[-1]
                     if (minHei < self.dataOut.heightList[0]):
                         minHei = self.dataOut.heightList[0]
                     if (maxHei > self.dataOut.heightList[-1]):
                         maxHei = self.dataOut.heightList[-1]
                     minIndex = 0
                     maxIndex = 0
                     heights = self.dataOut.heightList
                     inda = numpy.where(heights >= minHei)
                     indb = numpy.where(heights <= maxHei)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(heights)
                     self.selectHeightsByIndex(minIndex, maxIndex)
                     return 1
                 def selectHeightsByIndex(self, minIndex, maxIndex):
                     """
                     Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
                     minIndex <= index <= maxIndex
                     Input:
                         minIndex    :    valor de indice minimo de altura a considerar
                         maxIndex    :    valor de indice maximo de altura a considerar
                     Affected:
                         self.dataOut.data
                         self.dataOut.heightList
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if (minIndex < 0) or (minIndex > maxIndex):
                         raise ValueError("Height index range (%d,%d) is not valid" % (minIndex, maxIndex))
                     if (maxIndex >= self.dataOut.nHeights):
                         maxIndex = self.dataOut.nHeights
                     #voltage
                     if self.dataOut.flagDataAsBlock:
                         """
                         Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         data = self.dataOut.data[:,:, minIndex:maxIndex]
                     else:
                         data = self.dataOut.data[:, minIndex:maxIndex]
                     #         firstHeight = self.dataOut.heightList[minIndex]
                     self.dataOut.data = data
                     self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
                     if self.dataOut.nHeights <= 1:
                         raise ValueError("selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights))
                     return 1
                 def filterByHeights(self, window):
                     deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     if window == None:
                         window = (self.dataOut.radarControllerHeaderObj.txA/self.dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
                     newdelta = deltaHeight * window
                     r = self.dataOut.nHeights % window
                     newheights = (self.dataOut.nHeights-r)/window
                     if newheights <= 1:
                         raise ValueError("filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(self.dataOut.nHeights, window))
                     if self.dataOut.flagDataAsBlock:
                         """
                         Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         buffer = self.dataOut.data[:, :, 0:int(self.dataOut.nHeights-r)]
                         buffer = buffer.reshape(self.dataOut.nChannels, self.dataOut.nProfiles, int(self.dataOut.nHeights/window), window)
                         buffer = numpy.sum(buffer,3)
                     else:
                         buffer = self.dataOut.data[:,0:int(self.dataOut.nHeights-r)]
                         buffer = buffer.reshape(self.dataOut.nChannels,int(self.dataOut.nHeights/window),int(window))
                         buffer = numpy.sum(buffer,2)
                     self.dataOut.data = buffer
                     self.dataOut.heightList = self.dataOut.heightList[0] + numpy.arange( newheights )*newdelta
                     self.dataOut.windowOfFilter = window
                 def setH0(self, h0, deltaHeight = None):
                     if not deltaHeight:
                         deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     nHeights = self.dataOut.nHeights
                     newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
                     self.dataOut.heightList = newHeiRange
                 def deFlip(self, channelList = []):
                     data = self.dataOut.data.copy()
                     if self.dataOut.flagDataAsBlock:
                         flip = self.flip
                         profileList = list(range(self.dataOut.nProfiles))
                         if not channelList:
                             for thisProfile in profileList:
                                 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
                                 flip *= -1.0
                         else:
                             for thisChannel in channelList:
                                 if thisChannel not in self.dataOut.channelList:
                                     continue
                                 for thisProfile in profileList:
                                     data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
                                     flip *= -1.0
                         self.flip = flip
                     else:
                         if not channelList:
                             data[:,:] = data[:,:]*self.flip
                         else:
                             for thisChannel in channelList:
                                 if thisChannel not in self.dataOut.channelList:
                                     continue
                                 data[thisChannel,:] = data[thisChannel,:]*self.flip
                         self.flip *= -1.
                     self.dataOut.data = data
                 def setRadarFrequency(self, frequency=None):
                     if frequency != None:
                         self.dataOut.frequency = frequency
                     return 1
                 def interpolateHeights(self, topLim, botLim):
                     #69 al 72 para julia
                     #82-84 para meteoros
                     if len(numpy.shape(self.dataOut.data))==2:
                         sampInterp = (self.dataOut.data[:,botLim-1] + self.dataOut.data[:,topLim+1])/2
                         sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
                         #self.dataOut.data[:,botLim:limSup+1] = sampInterp
                         self.dataOut.data[:,botLim:topLim+1] = sampInterp
                     else:
                         nHeights = self.dataOut.data.shape[2]
                         x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
                         y = self.dataOut.data[:,:,list(range(botLim))+list(range(topLim+1,nHeights))]
                         f = interpolate.interp1d(x, y, axis = 2)
                         xnew = numpy.arange(botLim,topLim+1)
                         ynew = f(xnew)
                         self.dataOut.data[:,:,botLim:topLim+1]  = ynew
                 # import collections
             class CohInt(Operation):
                 isConfig = False
                 __profIndex = 0
                 __byTime = False
                 __initime = None
                 __lastdatatime = None
                 __integrationtime = None
                 __buffer = None
                 __bufferStride = []
                 __dataReady = False
                 __profIndexStride = 0
                 __dataToPutStride = False
                 n = None
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     #   self.isConfig = False
                 def setup(self, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False):
                     """
                     Set the parameters of the integration class.
                     Inputs:
                         n               :    Number of coherent integrations
                         timeInterval    :    Time of integration. If the parameter "n" is selected this one does not work
                         overlapping     :
                     """
                     self.__initime = None
                     self.__lastdatatime = 0
                     self.__buffer = None
                     self.__dataReady = False
                     self.byblock = byblock
                     self.stride = stride
                     if n == None and timeInterval == None:
                         raise ValueError("n or timeInterval should be specified ...")
                     if n != None:
                         self.n = n
                         self.__byTime = False
                     else:
                         self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
                         self.n = 9999
                         self.__byTime = True
                     if overlapping:
                         self.__withOverlapping = True
                         self.__buffer = None
                     else:
                         self.__withOverlapping = False
                         self.__buffer = 0
                     self.__profIndex = 0
                 def putData(self, data):
                     """
                     Add a profile to the __buffer and increase in one the __profileIndex
                     """
                     if not self.__withOverlapping:
                         self.__buffer += data.copy()
                         self.__profIndex += 1
                         return
                     #Overlapping data
                     nChannels, nHeis = data.shape
                     data = numpy.reshape(data, (1, nChannels, nHeis))
                     #If the buffer is empty then it takes the data value
                     if self.__buffer is None:
                         self.__buffer = data
                         self.__profIndex += 1
                         return
                     #If the buffer length is lower than n then stakcing the data value
                     if self.__profIndex < self.n:
                         self.__buffer = numpy.vstack((self.__buffer, data))
                         self.__profIndex += 1
                         return
                     #If the buffer length is equal to n then replacing the last buffer value with the data value
                     self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
                     self.__buffer[self.n-1] = data
                     self.__profIndex = self.n
                     return
                 def pushData(self):
                     """
                     Return the sum of the last profiles and the profiles used in the sum.
                     Affected:
                     self.__profileIndex
                     """
                     if not self.__withOverlapping:
                         data = self.__buffer
                         n = self.__profIndex
                         self.__buffer = 0
                         self.__profIndex = 0
                         return data, n
                     #Integration with Overlapping
                     data = numpy.sum(self.__buffer, axis=0)
                     # print data
                     # raise
                     n = self.__profIndex
                     return data, n
                 def byProfiles(self, data):
                     self.__dataReady = False
                     avgdata = None
                     #         n = None
                     # print data
                     # raise
                     self.putData(data)
                     if self.__profIndex == self.n:
                         avgdata, n = self.pushData()
                         self.__dataReady = True
                     return avgdata
                 def byTime(self, data, datatime):
                     self.__dataReady = False
                     avgdata = None
                     n = None
                     self.putData(data)
                     if (datatime - self.__initime) >= self.__integrationtime:
                         avgdata, n = self.pushData()
                         self.n = n
                         self.__dataReady = True
                     return avgdata
                 def integrateByStride(self, data, datatime):
                     # print data
                     if self.__profIndex == 0:
                         self.__buffer = [[data.copy(), datatime]]
                     else:
                         self.__buffer.append([data.copy(),datatime])
                     self.__profIndex += 1
                     self.__dataReady = False
                     if self.__profIndex == self.n * self.stride :
                         self.__dataToPutStride = True
                         self.__profIndexStride = 0
                         self.__profIndex = 0
                         self.__bufferStride = []
                         for i in range(self.stride):
                             current = self.__buffer[i::self.stride]
                             data = numpy.sum([t[0] for t in current], axis=0)
                             avgdatatime = numpy.average([t[1] for t in current])
                             # print data
                             self.__bufferStride.append((data, avgdatatime))
                     if self.__dataToPutStride:
                         self.__dataReady = True
                         self.__profIndexStride += 1
                         if self.__profIndexStride == self.stride:
                             self.__dataToPutStride = False
                         # print self.__bufferStride[self.__profIndexStride - 1]
                         # raise
                         return self.__bufferStride[self.__profIndexStride - 1]
                     return None, None
                 def integrate(self, data, datatime=None):
                     if self.__initime == None:
                         self.__initime = datatime
                     if self.__byTime:
                         avgdata = self.byTime(data, datatime)
                     else:
                         avgdata = self.byProfiles(data)
                     self.__lastdatatime = datatime
                     if avgdata is None:
                         return None, None
                     avgdatatime = self.__initime
                     deltatime = datatime - self.__lastdatatime
                     if not self.__withOverlapping:
                         self.__initime = datatime
                     else:
                         self.__initime += deltatime
                     return avgdata, avgdatatime
                 def integrateByBlock(self, dataOut):
                     times = int(dataOut.data.shape[1]/self.n)
                     avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
                     id_min = 0
                     id_max = self.n
                     for i in range(times):
                         junk = dataOut.data[:,id_min:id_max,:]
                         avgdata[:,i,:] = junk.sum(axis=1)
                         id_min += self.n
                         id_max += self.n
                     timeInterval = dataOut.ippSeconds*self.n
                     avgdatatime = (times - 1) * timeInterval + dataOut.utctime
                     self.__dataReady = True
                     return avgdata, avgdatatime
                 def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
                     if not self.isConfig:
                         self.setup(n=n, stride=stride, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
                         self.isConfig = True
                     if dataOut.flagDataAsBlock:
                         """
                         Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         avgdata, avgdatatime = self.integrateByBlock(dataOut)
                         dataOut.nProfiles /= self.n
                     else:
                         if stride is None:
                             avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
                         else:
                             avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
                     #   dataOut.timeInterval *= n
                     dataOut.flagNoData = True
                     if self.__dataReady:
                         dataOut.data = avgdata
                         dataOut.nCohInt *= self.n
                         dataOut.utctime = avgdatatime
                         # print avgdata, avgdatatime
                         # raise
                         #   dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
                         dataOut.flagNoData = False
                     return dataOut
             class Decoder(Operation):
                 isConfig = False
                 __profIndex = 0
                 code = None
                 nCode = None
                 nBaud = None
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.times = None
                     self.osamp = None
                 #         self.__setValues = False
                     self.isConfig = False
                     self.setupReq = False
                 def setup(self, code, osamp, dataOut):
                     self.__profIndex = 0
                     self.code = code
                     self.nCode = len(code)
                     self.nBaud = len(code[0])
                     if (osamp != None) and (osamp >1):
                         self.osamp = osamp
                         self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
                         self.nBaud = self.nBaud*self.osamp
                     self.__nChannels = dataOut.nChannels
                     self.__nProfiles = dataOut.nProfiles
                     self.__nHeis = dataOut.nHeights
                     if self.__nHeis < self.nBaud:
                         raise ValueError('Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud))
                     #Frequency
                     __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
                     __codeBuffer[:,0:self.nBaud] = self.code
                     self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
                     if dataOut.flagDataAsBlock:
                         self.ndatadec = self.__nHeis #- self.nBaud + 1
                         self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
                     else:
                         #Time
                         self.ndatadec = self.__nHeis #- self.nBaud + 1
                         self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
                 def __convolutionInFreq(self, data):
                     fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
                     fft_data = numpy.fft.fft(data, axis=1)
                     conv = fft_data*fft_code
                     data = numpy.fft.ifft(conv,axis=1)
                     return data
                 def __convolutionInFreqOpt(self, data):
                     raise NotImplementedError
                 def __convolutionInTime(self, data):
                     code = self.code[self.__profIndex]
                     for i in range(self.__nChannels):
                         self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
                     return self.datadecTime
                 def __convolutionByBlockInTime(self, data):
                     repetitions = int(self.__nProfiles / self.nCode)
                     junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
                     junk = junk.flatten()
                     code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
                     profilesList = range(self.__nProfiles)
                     for i in range(self.__nChannels):
                         for j in profilesList:
                             self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
                     return self.datadecTime
                 def __convolutionByBlockInFreq(self, data):
                     raise NotImplementedError("Decoder by frequency fro Blocks not implemented")
                     fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
                     fft_data = numpy.fft.fft(data, axis=2)
                     conv = fft_data*fft_code
                     data = numpy.fft.ifft(conv,axis=2)
                     return data
                 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
                     if dataOut.flagDecodeData:
                         print("This data is already decoded, recoding again ...")
                     if not self.isConfig:
                         if code is None:
                             if dataOut.code is None:
                                 raise ValueError("Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type)
                             code = dataOut.code
                         else:
                             code = numpy.array(code).reshape(nCode,nBaud)
                         self.setup(code, osamp, dataOut)
                         self.isConfig = True
                         if mode == 3:
                             sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
                         if times != None:
                             sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
                     if self.code is None:
                         print("Fail decoding: Code is not defined.")
                         return
                     self.__nProfiles = dataOut.nProfiles
                     datadec = None
                     if mode == 3:
                         mode = 0
                     if dataOut.flagDataAsBlock:
                         """
                         Decoding when data have been read as block,
                         """
                         if mode == 0:
                             datadec = self.__convolutionByBlockInTime(dataOut.data)
                         if mode == 1:
                             datadec = self.__convolutionByBlockInFreq(dataOut.data)
                     else:
                         """
                         Decoding when data have been read profile by profile
                         """
                         if mode == 0:
                             datadec = self.__convolutionInTime(dataOut.data)
                         if mode == 1:
                             datadec = self.__convolutionInFreq(dataOut.data)
                         if mode == 2:
                             datadec = self.__convolutionInFreqOpt(dataOut.data)
                     if datadec is None:
                         raise ValueError("Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode)
                     dataOut.code = self.code
                     dataOut.nCode = self.nCode
                     dataOut.nBaud = self.nBaud
                     dataOut.data = datadec
                     dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
                     dataOut.flagDecodeData = True #asumo q la data esta decodificada
                     if self.__profIndex == self.nCode-1:
                         self.__profIndex = 0
                         return dataOut
                     self.__profIndex += 1
                     return dataOut
                 #        dataOut.flagDeflipData = True #asumo q la data no esta sin flip
             class ProfileConcat(Operation):
                 isConfig = False
                 buffer = None
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.profileIndex = 0
                 def reset(self):
                     self.buffer = numpy.zeros_like(self.buffer)
                     self.start_index = 0
                     self.times = 1
                 def setup(self, data, m, n=1):
                     self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
                     self.nHeights = data.shape[1]#.nHeights
                     self.start_index = 0
                     self.times = 1
                 def concat(self, data):
                     self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
                     self.start_index = self.start_index + self.nHeights
                 def run(self, dataOut, m):
                     dataOut.flagNoData = True
                     if not self.isConfig:
                         self.setup(dataOut.data, m, 1)
                         self.isConfig = True
                     if dataOut.flagDataAsBlock:
                         raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
                     else:
                         self.concat(dataOut.data)
                         self.times += 1
                         if self.times > m:
                             dataOut.data = self.buffer
                             self.reset()
                             dataOut.flagNoData = False
                             # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
                             deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                             xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
                             dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
                             dataOut.ippSeconds *= m
                     return dataOut
             class ProfileSelector(Operation):
                 profileIndex = None
                 # Tamanho total de los perfiles
                 nProfiles = None
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.profileIndex = 0
                 def incProfileIndex(self):
                     self.profileIndex += 1
                     if self.profileIndex >= self.nProfiles:
                         self.profileIndex = 0
                 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
                     if profileIndex < minIndex:
                         return False
                     if profileIndex > maxIndex:
                         return False
                     return True
                 def isThisProfileInList(self, profileIndex, profileList):
                     if profileIndex not in profileList:
                         return False
                     return True
                 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
                     """
                     ProfileSelector:
                     Inputs:
                         profileList        :    Index of profiles selected. Example: profileList = (0,1,2,7,8)
                         profileRangeList    :    Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
                         rangeList            :    List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
                     """
                     if rangeList is not None:
                         if type(rangeList[0]) not in (tuple, list):
                             rangeList = [rangeList]
                     dataOut.flagNoData = True
                     if dataOut.flagDataAsBlock:
                         """
                         data dimension  = [nChannels, nProfiles, nHeis]
                         """
                         if profileList != None:
                             dataOut.data = dataOut.data[:,profileList,:]
                         if profileRangeList != None:
                             minIndex = profileRangeList[0]
                             maxIndex = profileRangeList[1]
                             profileList = list(range(minIndex, maxIndex+1))
                             dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
                         if rangeList != None:
                             profileList = []
                             for thisRange in rangeList:
                                 minIndex = thisRange[0]
                                 maxIndex = thisRange[1]
                                 profileList.extend(list(range(minIndex, maxIndex+1)))
                             dataOut.data = dataOut.data[:,profileList,:]
                         dataOut.nProfiles = len(profileList)
                         dataOut.profileIndex = dataOut.nProfiles - 1
                         dataOut.flagNoData = False
                         return dataOut
                     """
                     data dimension  = [nChannels, nHeis]
                     """
                     if profileList != None:
                         if self.isThisProfileInList(dataOut.profileIndex, profileList):
                             self.nProfiles = len(profileList)
                             dataOut.nProfiles = self.nProfiles
                             dataOut.profileIndex = self.profileIndex
                             dataOut.flagNoData = False
                             self.incProfileIndex()
                         return dataOut
                     if profileRangeList != None:
                         minIndex = profileRangeList[0]
                         maxIndex = profileRangeList[1]
                         if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
                             self.nProfiles = maxIndex - minIndex + 1
                             dataOut.nProfiles = self.nProfiles
                             dataOut.profileIndex = self.profileIndex
                             dataOut.flagNoData = False
                             self.incProfileIndex()
                         return dataOut
                     if rangeList != None:
                         nProfiles = 0
                         for thisRange in rangeList:
                             minIndex = thisRange[0]
                             maxIndex = thisRange[1]
                             nProfiles += maxIndex - minIndex + 1
                         for thisRange in rangeList:
                             minIndex = thisRange[0]
                             maxIndex = thisRange[1]
                             if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
                                 self.nProfiles = nProfiles
                                 dataOut.nProfiles = self.nProfiles
                                 dataOut.profileIndex = self.profileIndex
                                 dataOut.flagNoData = False
                                 self.incProfileIndex()
                                 break
                         return dataOut
                     if beam != None: #beam is only for AMISR data
                         if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
                             dataOut.flagNoData = False
                             dataOut.profileIndex = self.profileIndex
                             self.incProfileIndex()
                         return dataOut
                     raise ValueError("ProfileSelector needs profileList, profileRangeList or rangeList parameter")
                     #return False
                     return dataOut
             class Reshaper(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.__buffer = None
                     self.__nitems = 0
                 def __appendProfile(self, dataOut, nTxs):
                     if self.__buffer is None:
                         shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
                         self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
                     ini = dataOut.nHeights * self.__nitems
                     end = ini + dataOut.nHeights
                     self.__buffer[:, ini:end] = dataOut.data
                     self.__nitems += 1
                     return int(self.__nitems*nTxs)
                 def __getBuffer(self):
                     if self.__nitems == int(1./self.__nTxs):
                         self.__nitems = 0
                         return self.__buffer.copy()
                     return None
                 def __checkInputs(self, dataOut, shape, nTxs):
                     if shape is None and nTxs is None:
                         raise ValueError("Reshaper: shape of factor should be defined")
                     if nTxs:
                         if nTxs < 0:
                             raise ValueError("nTxs should be greater than 0")
                         if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
                             raise ValueError("nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs)))
                         shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
                         return shape, nTxs
                     if len(shape) != 2 and len(shape) !=  3:
                         raise ValueError("shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights))
                     if len(shape) == 2:
                         shape_tuple = [dataOut.nChannels]
                         shape_tuple.extend(shape)
                     else:
                         shape_tuple = list(shape)
                     nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
                     return shape_tuple, nTxs
                 def run(self, dataOut, shape=None, nTxs=None):
                     shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
                     dataOut.flagNoData = True
                     profileIndex = None
                     if dataOut.flagDataAsBlock:
                         dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
                         dataOut.flagNoData = False
                         profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
                     else:
                         if self.__nTxs < 1:
                             self.__appendProfile(dataOut, self.__nTxs)
                             new_data = self.__getBuffer()
                             if new_data is not None:
                                 dataOut.data = new_data
                                 dataOut.flagNoData = False
                                 profileIndex = dataOut.profileIndex*nTxs
                         else:
                             raise ValueError("nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)")
                     deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                     dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
                     dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
                     dataOut.profileIndex = profileIndex
                     dataOut.ippSeconds /= self.__nTxs
                     return dataOut
             class SplitProfiles(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                 def run(self, dataOut, n):
                     dataOut.flagNoData = True
                     profileIndex = None
                     if dataOut.flagDataAsBlock:
                         #nchannels, nprofiles, nsamples
                         shape = dataOut.data.shape
                         if shape[2] % n != 0:
                             raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
                         new_shape = shape[0], shape[1]*n, int(shape[2]/n)
                         dataOut.data = numpy.reshape(dataOut.data, new_shape)
                         dataOut.flagNoData = False
                         profileIndex = int(dataOut.nProfiles/n) - 1
                     else:
                         raise ValueError("Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)")
                     deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                     dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
                     dataOut.nProfiles = int(dataOut.nProfiles*n)
                     dataOut.profileIndex = profileIndex
                     dataOut.ippSeconds /= n
                     return dataOut
             class CombineProfiles(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.__remData = None
                     self.__profileIndex = 0
                 def run(self, dataOut, n):
                     dataOut.flagNoData = True
                     profileIndex = None
                     if dataOut.flagDataAsBlock:
                         #nchannels, nprofiles, nsamples
                         shape = dataOut.data.shape
                         new_shape = shape[0], shape[1]/n, shape[2]*n
                         if shape[1] % n != 0:
                             raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[1]))
                         dataOut.data = numpy.reshape(dataOut.data, new_shape)
                         dataOut.flagNoData = False
                         profileIndex = int(dataOut.nProfiles*n) - 1
                     else:
                         #nchannels, nsamples
                         if self.__remData is None:
                             newData = dataOut.data
                         else:
                             newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
                         self.__profileIndex += 1
                         if self.__profileIndex < n:
                             self.__remData = newData
                             #continue
                             return
                         self.__profileIndex = 0
                         self.__remData = None
                         dataOut.data = newData
                         dataOut.flagNoData = False
                         profileIndex = dataOut.profileIndex/n
                     deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                     dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
                     dataOut.nProfiles = int(dataOut.nProfiles/n)
                     dataOut.profileIndex = profileIndex
                     dataOut.ippSeconds *= n
                     return dataOut
             # import collections
             # from scipy.stats import mode
             #
             # class Synchronize(Operation):
             #
             #     isConfig = False
             #     __profIndex = 0
             #
             #     def __init__(self, **kwargs):
             #
             #         Operation.__init__(self, **kwargs)
             # #         self.isConfig = False
             #         self.__powBuffer = None
             #         self.__startIndex = 0
             #         self.__pulseFound = False
             #
             #     def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
             #
             #         #Read data
             #
             #         powerdB = dataOut.getPower(channel = channel)
             #         noisedB = dataOut.getNoise(channel = channel)[0]
             #
             #         self.__powBuffer.extend(powerdB.flatten())
             #
             #         dataArray = numpy.array(self.__powBuffer)
             #
             #         filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
             #
             #         maxValue = numpy.nanmax(filteredPower)
             #
             #         if maxValue < noisedB + 10:
             #             #No se encuentra ningun pulso de transmision
             #             return None
             #
             #         maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
             #
             #         if len(maxValuesIndex) < 2:
             #             #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
             #             return None
             #
             #         phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
             #
             #         #Seleccionar solo valores con un espaciamiento de nSamples
             #         pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
             #
             #         if len(pulseIndex) < 2:
             #             #Solo se encontro un pulso de transmision con ancho mayor a 1
             #             return None
             #
             #         spacing = pulseIndex[1:] - pulseIndex[:-1]
             #
             #         #remover senales que se distancien menos de 10 unidades o muestras
             #         #(No deberian existir IPP menor a 10 unidades)
             #
             #         realIndex = numpy.where(spacing > 10 )[0]
             #
             #         if len(realIndex) < 2:
             #             #Solo se encontro un pulso de transmision con ancho mayor a 1
             #             return None
             #
             #         #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
             #         realPulseIndex = pulseIndex[realIndex]
             #
             #         period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
             #
             #         print "IPP = %d samples" %period
             #
             #         self.__newNSamples = dataOut.nHeights #int(period)
             #         self.__startIndex = int(realPulseIndex[0])
             #
             #         return 1
             #
             #
             #     def setup(self, nSamples, nChannels, buffer_size = 4):
             #
             #         self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
             #                                           maxlen = buffer_size*nSamples)
             #
             #         bufferList = []
             #
             #         for i in range(nChannels):
             #             bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) +  numpy.NAN,
             #                                           maxlen = buffer_size*nSamples)
             #
             #             bufferList.append(bufferByChannel)
             #
             #         self.__nSamples = nSamples
             #         self.__nChannels = nChannels
             #         self.__bufferList = bufferList
             #
             #     def run(self, dataOut, channel = 0):
             #
             #         if not self.isConfig:
             #             nSamples = dataOut.nHeights
             #             nChannels = dataOut.nChannels
             #             self.setup(nSamples, nChannels)
             #             self.isConfig = True
             #
             #         #Append new data to internal buffer
             #         for thisChannel in range(self.__nChannels):
             #             bufferByChannel = self.__bufferList[thisChannel]
             #             bufferByChannel.extend(dataOut.data[thisChannel])
             #
             #         if self.__pulseFound:
             #             self.__startIndex -= self.__nSamples
             #
             #         #Finding Tx Pulse
             #         if not self.__pulseFound:
             #             indexFound = self.__findTxPulse(dataOut, channel)
             #
             #             if indexFound == None:
             #                 dataOut.flagNoData = True
             #                 return
             #
             #             self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
             #             self.__pulseFound = True
             #             self.__startIndex = indexFound
             #
             #         #If pulse was found ...
             #         for thisChannel in range(self.__nChannels):
             #             bufferByChannel = self.__bufferList[thisChannel]
             #             #print self.__startIndex
             #             x = numpy.array(bufferByChannel)
             #             self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
             #
             #         deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
             #         dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
             # #             dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
             #
             #         dataOut.data = self.__arrayBuffer
             #
             #         self.__startIndex += self.__newNSamples
             #
             #         return

schainpy/model/utils/jroutils_ftp.py 0 0

NO CONTENT: modified file

schainpy/model/utils/jroutils_publish.py 0 0

NO CONTENT: modified file

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages