schain Commit - r1241:c3044f867269 · Jicamarca Repository

Errors handling and gracefully terminate main process

jespinoza -

r1241:c3044f867269

parent child

Context file:

r1241:c3044f867269

Collapse all files

The requested changes are too big and content was truncated. Show full diff

schainpy/admin.py +1 -2

             """
             The admin module contains all administrative classes relating to the schain python api.
             The main role of this module is to send some reports. It contains a
             notification class and a standard error handing class.
             $Id: admin.py 3966 2015-12-01 14:32:29Z miguel.urco $
             """
             import os
             import sys
             import time
             import traceback
             import smtplib
             if sys.version[0] == '3':
                 from configparser import ConfigParser
             else:
                 from ConfigParser import ConfigParser
             import io
             from threading import Thread
             from multiprocessing import Process
             from email.mime.text import MIMEText
             from email.mime.application import MIMEApplication
             from email.mime.multipart import MIMEMultipart
             import schainpy
             from schainpy.utils import log
             from schainpy.model.graphics.jroplot_base import popup
             def get_path():
                 '''
                 Return schainpy path
                 '''
                 try:
                     root = __file__
                     if os.path.islink(root):
                         root = os.path.realpath(root)
                     return os.path.dirname(os.path.abspath(root))
                 except:
                     log.error('I am sorry, but something is wrong... __file__ not found')
             class Alarm(Process):
                 '''
                 modes:
 - All
 - Send email
 - Popup message
 - Sound alarm
 - Send to alarm system TODO
                 '''
                 def __init__(self, modes=[], **kwargs):
                     Process.__init__(self)
                     self.modes = modes
                     self.kwargs = kwargs
                 @staticmethod
                 def play_sound():
                     sound = os.path.join(get_path(), 'alarm1.oga')
                     if os.path.exists(sound):
                         for __ in range(2):
                             os.system('paplay {}'.format(sound))
                             time.sleep(0.5)
                     else:
                         log.warning('Unable to play alarm, sound file not found', 'ADMIN')
                 @staticmethod
                 def send_email(**kwargs):
                     notifier = SchainNotify()
-                    print(kwargs)
                     notifier.notify(**kwargs)
                 @staticmethod
                 def show_popup(message):
                     if isinstance(message, list):
                         message = message[-1]
                     popup(message)
                 @staticmethod
                 def send_alarm():
                     pass
                 @staticmethod
                 def get_kwargs(kwargs, keys):
                     ret = {}
                     for key in keys:
                         ret[key] = kwargs[key]
                     return ret
                 def run(self):
                     tasks = {
 : self.send_email,
 : self.show_popup,
 : self.play_sound,
 : self.send_alarm,
                     }
                     tasks_args = {
 : ['email', 'message', 'subject', 'subtitle', 'filename'],
 : ['message'],
 : [],
 : [],
                     }
                     procs = []
                     for mode in self.modes:
                         if 0 in self.modes:
                             for x in tasks:
                                 t = Thread(target=tasks[x], kwargs=self.get_kwargs(self.kwargs, tasks_args[x]))
                                 t.start()
                                 procs.append(t)
                             break
                         else:
                             t = Thread(target=tasks[mode], kwargs=self.get_kwargs(self.kwargs, tasks_args[mode]))
                             t.start()
                             procs.append(t)
                     for t in procs:
                         t.join()
             class SchainConfigure():
                 __DEFAULT_ADMINISTRATOR_EMAIL = "juan.espinoza@jro.igp.gob.pe"
                 __DEFAULT_EMAIL_SERVER = "jro-zimbra.igp.gob.pe"
                 __DEFAULT_SENDER_EMAIL = "notifier-schain@jro.igp.gob.pe"
                 __DEFAULT_SENDER_PASS = ""
                 __SCHAIN_ADMINISTRATOR_EMAIL = "CONTACT"
                 __SCHAIN_EMAIL_SERVER = "MAILSERVER"
                 __SCHAIN_SENDER_EMAIL = "MAILSERVER_ACCOUNT"
                 __SCHAIN_SENDER_PASS = "MAILSERVER_PASSWORD"
                 def __init__(self, initFile = None):
                     # Set configuration file
                     if (initFile == None):
                         self.__confFilePath = "/etc/schain.conf"
                     else:
                         self.__confFilePath = initFile
                     # open configuration file
                     try:
                         self.__confFile = open(self.__confFilePath, "r")
                     except IOError:
                         # can't read from file - use all hard-coded values
                         self.__initFromHardCode()
                         return
                     # create Parser using standard module ConfigParser
                     self.__parser = ConfigParser()
                     # read conf file into a StringIO with "[madrigal]\n" section heading prepended
                     strConfFile = io.StringIO("[schain]\n" + self.__confFile.read())
                     # parse StringIO configuration file
                     self.__parser.readfp(strConfFile)
                     # read information from configuration file
                     self.__readConfFile()
                     # close conf file
                     self.__confFile.close()
                 def __initFromHardCode(self):
                     self.__sender_email = self.__DEFAULT_SENDER_EMAIL
                     self.__sender_pass = self.__DEFAULT_SENDER_PASS
                     self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
                     self.__email_server = self.__DEFAULT_EMAIL_SERVER
                 def __readConfFile(self):
                     """__readConfFile is a private helper function that reads information from the parsed config file.
                     Inputs: None
                     Returns: Void.
                     Affects: Initializes class member variables that are found in the config file.
                     Exceptions: MadrigalError thrown if any key not found.
                     """
                     # get the sender email
                     try:
                         self.__sender_email = self.__parser.get("schain", self.__SCHAIN_SENDER_EMAIL)
                     except:
                         self.__sender_email = self.__DEFAULT_SENDER_EMAIL
                     # get the sender password
                     try:
                         self.__sender_pass = self.__parser.get("schain", self.__SCHAIN_SENDER_PASS)
                     except:
                         self.__sender_pass = self.__DEFAULT_SENDER_PASS
                     # get the administrator email
                     try:
                         self.__admin_email = self.__parser.get("schain", self.__SCHAIN_ADMINISTRATOR_EMAIL)
                     except:
                         self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
                     # get the server email
                     try:
                         self.__email_server = self.__parser.get("schain", self.__SCHAIN_EMAIL_SERVER)
                     except:
                         self.__email_server = self.__DEFAULT_EMAIL_SERVER
                 def getEmailServer(self):
                     return self.__email_server
                 def getSenderEmail(self):
                     return self.__sender_email
                 def getSenderPass(self):
                     return self.__sender_pass
                 def getAdminEmail(self):
                     return self.__admin_email
             class SchainNotify:
                 """SchainNotify is an object used to send messages to an administrator about a Schain software.
                 This object provides functions needed to send messages to an administrator about a Schain , for now
                 only sendAlert, which sends an email to the site administrator found is ADMIN_EMAIL
                 Usage example:
                     import schainpy.admin
                     try:
                         adminObj =  schainpy.admin.SchainNotify()
                         adminObj.sendAlert('This is important!', 'Important Message')
                     except schainpy.admin.SchainError, e:
                         print e.getExceptionStr()
                 Non-standard Python modules used:
                 None
                 Exceptions thrown: None - Note that SchainNotify tries every trick it knows to avoid
                 throwing exceptions, since this is the class that will generally be called when there is a problem.
                 Change history:
                 Written by "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe  Dec. 1, 2015
                 """
                 #constants
                 def __init__(self):
                     """__init__ initializes SchainNotify by getting some basic information from SchainDB and SchainSite.
                     Note that SchainNotify tries every trick it knows to avoid throwing exceptions, since
                     this is the class that will generally be called when there is a problem.
                     Inputs: Existing SchainDB object, by default = None.
                     Returns: void
                     Affects: Initializes self.__binDir.
                     Exceptions: None.
                     """
                     # note that the main configuration file is unavailable
                     # the best that can be done is send an email to root using localhost mailserver
                     confObj = SchainConfigure()
                     self.__emailFromAddress = confObj.getSenderEmail()
                     self.__emailPass = confObj.getSenderPass()
                     self.__emailToAddress = confObj.getAdminEmail()
                     self.__emailServer  = confObj.getEmailServer()
                 def sendEmail(self, email_from, email_to, subject='Error running ...', message="", subtitle="", filename="", html_format=True):
                     if not email_to:
                         return 0
                     if not self.__emailServer:
                         return 0
                     log.success('Sending email to {}...'.format(email_to), 'System')
                     msg = MIMEMultipart()
                     msg['Subject'] = subject
-                    msg['From'] = "(Python SChain API): " + email_from
+                    msg['From'] = "SChain API (v{}) <{}>".format(schainpy.__version__, email_from)
                     msg['Reply-to'] = email_from
                     msg['To'] = email_to
                     # That is what u see if dont have an email reader:
                     msg.preamble = 'SChainPy'
                     if html_format:
                         message = "<h1> %s </h1>" %subject + "<h3>" + subtitle.replace("\n", "</h3><h3>\n") + "</h3>" + message.replace("\n", "<br>\n")
                         message = "<html>\n" + message + '</html>'
                         # This is the textual part:
                         part = MIMEText(message, "html")
                     else:
                         message = subject + "\n" + subtitle + "\n" + message
                         part = MIMEText(message)
                     msg.attach(part)
                     if filename and os.path.isfile(filename):
                         # This is the binary part(The Attachment):
                         part = MIMEApplication(open(filename,"rb").read())
                         part.add_header('Content-Disposition',
                                         'attachment',
                                         filename=os.path.basename(filename))
                         msg.attach(part)
                     # Create an instance in SMTP server
                     try:
                         smtp = smtplib.SMTP(self.__emailServer)
                     except:
                         log.error('Could not connect to server {}'.format(self.__emailServer), 'System')
                         return 0
                     # Start the server:
                 #         smtp.ehlo()
                     if self.__emailPass:
                         smtp.login(self.__emailFromAddress, self.__emailPass)
                     # Send the email
                     try:
                         smtp.sendmail(msg['From'], msg['To'], msg.as_string())
                     except:
                         log.error('Could not send the email to {}'.format(msg['To']), 'System')
                         smtp.quit()
                         return 0
                     smtp.quit()
                     log.success('Email sent ', 'System')
                     return 1
                 def sendAlert(self, message, subject = "", subtitle="", filename=""):
                     """sendAlert sends an email with the given message and optional title.
                     Inputs: message (string), and optional title (string)
                     Returns: void
                     Affects: none
                     Exceptions: None.
                     """
                     if not self.__emailToAddress:
                         return 0
                     print("***** Sending alert to %s *****" %self.__emailToAddress)
                     # set up message
                     sent=self.sendEmail(email_from=self.__emailFromAddress,
                                        email_to=self.__emailToAddress,
                                        subject=subject,
                                        message=message,
                                        subtitle=subtitle,
                                        filename=filename)
                     if not sent:
                         return 0
                     return 1
                 def notify(self, email, message, subject = "", subtitle="", filename=""):
                     """notify sends an email with the given message and title to email.
                     Inputs: email (string), message (string), and subject (string)
                     Returns: void
                     Affects: none
                     Exceptions: None.
                     """
                     if email is None:
                         email = self.__emailToAddress
                     self.sendEmail(
                         email_from=self.__emailFromAddress,
                         email_to=email,
                         subject=subject,
                         message=message,
                         subtitle=subtitle,
                         filename=filename
                         )
             class SchainError(Exception):
                 """SchainError is an exception class that is thrown for all known errors using Schain Py lib.
                 Usage example:
                     import sys, traceback
                     import schainpy.admin
                     try:
                         test = open('ImportantFile.txt', 'r')
                     except:
                         raise schainpy.admin.SchainError('ImportantFile.txt not opened!',
                                                             traceback.format_exception(sys.exc_info()[0],
                                                                                     sys.exc_info()[1],
                                                                                     sys.exc_info()[2]))
                 """
                 def __init__(self, strInterpretation, exceptionList=None):
                     """ __init__ gathers the interpretation string along with all information from sys.exc_info().
                     Inputs:
                             strIntepretation - A string representing the programmer's interpretation of
                             why the exception occurred
                             exceptionList - a list of strings completely describing the exception.
                             Generated by traceback.format_exception(sys.exc_info()[0],
                                                                     sys.exc_info()[1],
                                                                     sys.exc_info()[2])
                     Returns: Void.
                     Affects: Initializes class member variables _strInterp, _strExcList.
                     Exceptions: None.
                     """
                     if not exceptionList:
                         exceptionList = traceback.format_exception(sys.exc_info()[0],
                                                                     sys.exc_info()[1],
                                                                     sys.exc_info()[2])
                     self._strInterp = strInterpretation
                     self._strExcList = exceptionList
                 def getExceptionStr(self):
                     """ getExceptionStr returns a formatted string ready for printing completely describing the exception.
                     Inputs: None
                     Returns: A formatted string ready for printing completely describing the exception.
                     Affects: None
                     Exceptions: None.
                     """
                     excStr = ''
                     excStr = excStr + self._strInterp + '\n\n'
                     if self._strExcList != None:
                         for item in self._strExcList:
                             excStr = excStr + str(item) + '\n'
                     return excStr
                 def __str__(self):
                     return(self.getExceptionStr())
                 def getExceptionHtml(self):
                     """ getExceptionHtml returns an Html formatted string completely describing the exception.
                     Inputs: None
                     Returns: A formatted string ready for printing completely describing the exception.
                     Affects: None
                     Exceptions: None.
                     """
                     excStr = '<BR>The following Schain Python exception has occurred:\n<BR>'
                     excStr = excStr + self._strInterp + '\n<BR>\n'
                     if self._strExcList != None:
                         for item in self._strExcList:
                             excStr = excStr + str(item) + '\n<BR>'
                     return excStr
             class SchainWarning(Exception):
                 pass
             if __name__ == '__main__':
                 test = SchainNotify()
                 test.sendAlert('This is a message from the python module SchainNotify', 'Test from SchainNotify')
                 print('Hopefully message sent - check.')

schainpy/controller.py +70 -42

             '''
             Updated on January , 2018, for multiprocessing purposes
             Author: Sergio Cortez
             Created on September , 2012
             '''
             from platform import python_version
             import sys
             import ast
             import datetime
             import traceback
             import math
             import time
             import zmq
-            from multiprocessing import Process, cpu_count
+            from multiprocessing import Process, Queue, cpu_count
             from threading import Thread
             from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
             from xml.dom import minidom
             from schainpy.admin import Alarm, SchainWarning
             from schainpy.model import *
             from schainpy.utils import log
             DTYPES = {
                 'Voltage': '.r',
                 'Spectra': '.pdata'
             }
             def MPProject(project, n=cpu_count()):
                 '''
                 Project wrapper to run schain in n processes
                 '''
                 rconf = project.getReadUnitObj()
                 op = rconf.getOperationObj('run')
                 dt1 = op.getParameterValue('startDate')
                 dt2 = op.getParameterValue('endDate')
                 tm1 = op.getParameterValue('startTime')
                 tm2 = op.getParameterValue('endTime')
                 days = (dt2 - dt1).days
                 for day in range(days + 1):
                     skip = 0
                     cursor = 0
                     processes = []
                     dt = dt1 + datetime.timedelta(day)
                     dt_str = dt.strftime('%Y/%m/%d')
                     reader = JRODataReader()
                     paths, files = reader.searchFilesOffLine(path=rconf.path,
                                                              startDate=dt,
                                                              endDate=dt,
                                                              startTime=tm1,
                                                              endTime=tm2,
                                                              ext=DTYPES[rconf.datatype])
                     nFiles = len(files)
                     if nFiles == 0:
                         continue
                     skip = int(math.ceil(nFiles / n))
                     while nFiles > cursor * skip:
                         rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
                                      skip=skip)
                         p = project.clone()
                         p.start()
                         processes.append(p)
                         cursor += 1
                     def beforeExit(exctype, value, trace):
                         for process in processes:
                             process.terminate()
                             process.join()
                         print(traceback.print_tb(trace))
                     sys.excepthook = beforeExit
                     for process in processes:
                         process.join()
                         process.terminate()
                     time.sleep(3)
             def wait(context):
                 time.sleep(1)
                 c = zmq.Context()
                 receiver = c.socket(zmq.SUB)
                 receiver.connect('ipc:///tmp/schain_{}_pub'.format(self.id))
                 receiver.setsockopt(zmq.SUBSCRIBE, self.id.encode())
                 msg = receiver.recv_multipart()[1]
                 context.terminate()
             class ParameterConf():
                 id = None
                 name = None
                 value = None
                 format = None
                 __formated_value = None
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.format = 'str'
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getValue(self):
                     value = self.value
                     format = self.format
                     if self.__formated_value != None:
                         return self.__formated_value
                     if format == 'obj':
                         return value
                     if format == 'str':
                         self.__formated_value = str(value)
                         return self.__formated_value
                     if value == '':
                         raise ValueError('%s: This parameter value is empty' % self.name)
                     if format == 'list':
                         strList = [s.strip() for s in value.split(',')]
                         self.__formated_value = strList
                         return self.__formated_value
                     if format == 'intlist':
                         '''
                         Example:
                             value = (0,1,2)
                         '''
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             new_value = [int(new_value)]
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'floatlist':
                         '''
                         Example:
                             value = (0.5, 1.4, 2.7)
                         '''
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             new_value = [float(new_value)]
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'date':
                         strList = value.split('/')
                         intList = [int(x) for x in strList]
                         date = datetime.date(intList[0], intList[1], intList[2])
                         self.__formated_value = date
                         return self.__formated_value
                     if format == 'time':
                         strList = value.split(':')
                         intList = [int(x) for x in strList]
                         time = datetime.time(intList[0], intList[1], intList[2])
                         self.__formated_value = time
                         return self.__formated_value
                     if format == 'pairslist':
                         '''
                         Example:
                             value = (0,1),(1,2)
                         '''
                         new_value = ast.literal_eval(value)
                         if type(new_value) not in (tuple, list):
                             raise ValueError('%s has to be a tuple or list of pairs' % value)
                         if type(new_value[0]) not in (tuple, list):
                             if len(new_value) != 2:
                                 raise ValueError('%s has to be a tuple or list of pairs' % value)
                             new_value = [new_value]
                         for thisPair in new_value:
                             if len(thisPair) != 2:
                                 raise ValueError('%s has to be a tuple or list of pairs' % value)
                         self.__formated_value = new_value
                         return self.__formated_value
                     if format == 'multilist':
                         '''
                         Example:
                             value = (0,1,2),(3,4,5)
                         '''
                         multiList = ast.literal_eval(value)
                         if type(multiList[0]) == int:
                             multiList = ast.literal_eval('(' + value + ')')
                         self.__formated_value = multiList
                         return self.__formated_value
                     if format == 'bool':
                         value = int(value)
                     if format == 'int':
                         value = float(value)
                     format_func = eval(format)
                     self.__formated_value = format_func(value)
                     return self.__formated_value
                 def updateId(self, new_id):
                     self.id = str(new_id)
                 def setup(self, id, name, value, format='str'):
                     self.id = str(id)
                     self.name = name
                     if format == 'obj':
                         self.value = value
                     else:
                         self.value = str(value)
                     self.format = str.lower(format)
                     self.getValue()
                     return 1
                 def update(self, name, value, format='str'):
                     self.name = name
                     self.value = str(value)
                     self.format = format
                 def makeXml(self, opElement):
                     if self.name not in ('queue',):
                         parmElement = SubElement(opElement, self.ELEMENTNAME)
                         parmElement.set('id', str(self.id))
                         parmElement.set('name', self.name)
                         parmElement.set('value', self.value)
                         parmElement.set('format', self.format)
                 def readXml(self, parmElement):
                     self.id = parmElement.get('id')
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                     self.format = str.lower(parmElement.get('format'))
                     # Compatible with old signal chain version
                     if self.format == 'int' and self.name == 'idfigure':
                         self.name = 'id'
                 def printattr(self):
                     print('Parameter[%s]: name = %s, value = %s, format = %s, project_id = %s' % (self.id, self.name, self.value, self.format, self.project_id))
             class OperationConf():
                 ELEMENTNAME = 'Operation'
                 def __init__(self):
                     self.id = '0'
                     self.name = None
                     self.priority = None
                     self.topic = None
                 def __getNewId(self):
                     return int(self.id) * 10 + len(self.parmConfObjList) + 1
                 def getId(self):
                     return self.id
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     n = 1
                     for parmObj in self.parmConfObjList:
                         idParm = str(int(new_id) * 10 + n)
                         parmObj.updateId(idParm)
                         n += 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getParameterObjList(self):
                     return self.parmConfObjList
                 def getParameterObj(self, parameterName):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.name != parameterName:
                             continue
                         return parmConfObj
                     return None
                 def getParameterObjfromValue(self, parameterValue):
                     for parmConfObj in self.parmConfObjList:
                         if parmConfObj.getValue() != parameterValue:
                             continue
                         return parmConfObj.getValue()
                     return None
                 def getParameterValue(self, parameterName):
                     parameterObj = self.getParameterObj(parameterName)
                 #         if not parameterObj:
                 #             return None
                     value = parameterObj.getValue()
                     return value
                 def getKwargs(self):
                     kwargs = {}
                     for parmConfObj in self.parmConfObjList:
                         if self.name == 'run' and parmConfObj.name == 'datatype':
                             continue
                         kwargs[parmConfObj.name] = parmConfObj.getValue()
                     return kwargs
-                def setup(self, id, name, priority, type, project_id):
+                def setup(self, id, name, priority, type, project_id, err_queue):
                     self.id = str(id)
                     self.project_id = project_id
                     self.name = name
                     self.type = type
                     self.priority = priority
+                    self.err_queue = err_queue
                     self.parmConfObjList = []
                 def removeParameters(self):
                     for obj in self.parmConfObjList:
                         del obj
                     self.parmConfObjList = []
                 def addParameter(self, name, value, format='str'):
                     if value is None:
                         return None
                     id = self.__getNewId()
                     parmConfObj = ParameterConf()
                     if not parmConfObj.setup(id, name, value, format):
                         return None
                     self.parmConfObjList.append(parmConfObj)
                     return parmConfObj
                 def changeParameter(self, name, value, format='str'):
                     parmConfObj = self.getParameterObj(name)
                     parmConfObj.update(name, value, format)
                     return parmConfObj
                 def makeXml(self, procUnitElement):
                     opElement = SubElement(procUnitElement, self.ELEMENTNAME)
                     opElement.set('id', str(self.id))
                     opElement.set('name', self.name)
                     opElement.set('type', self.type)
                     opElement.set('priority', str(self.priority))
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.makeXml(opElement)
                 def readXml(self, opElement, project_id):
                     self.id = opElement.get('id')
                     self.name = opElement.get('name')
                     self.type = opElement.get('type')
                     self.priority = opElement.get('priority')
                     self.project_id = str(project_id)
                     # Compatible with old signal chain version
                     # Use of 'run' method instead 'init'
                     if self.type == 'self' and self.name == 'init':
                         self.name = 'run'
                     self.parmConfObjList = []
                     parmElementList = opElement.iter(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         # Compatible with old signal chain version
                         # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
                         if self.type != 'self' and self.name == 'Plot':
                             if parmConfObj.format == 'str' and parmConfObj.name == 'type':
                                 self.name = parmConfObj.value
                                 continue
                         self.parmConfObjList.append(parmConfObj)
                 def printattr(self):
                     print('%s[%s]: name = %s, type = %s, priority = %s, project_id = %s' % (self.ELEMENTNAME,
                                                                            self.id,
                                                                            self.name,
                                                                            self.type,
                                                                            self.priority,
                                                                            self.project_id))
                     for parmConfObj in self.parmConfObjList:
                         parmConfObj.printattr()
                 def createObject(self):
                     className = eval(self.name)
                     if self.type == 'other':
                         opObj = className()
                     elif self.type == 'external':
                         kwargs = self.getKwargs()
-                        opObj = className(self.id, self.project_id, **kwargs)
+                        opObj = className(self.id, self.id, self.project_id, self.err_queue, 'Operation', **kwargs)
                         opObj.start()
+                        self.opObj = opObj
                     return opObj
             class ProcUnitConf():
                 ELEMENTNAME = 'ProcUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.opConfObjList = []
                     self.procUnitObj = None
                     self.opObjDict = {}
                 def __getPriority(self):
                     return len(self.opConfObjList) + 1
                 def __getNewId(self):
                     return int(self.id) * 10 + len(self.opConfObjList) + 1
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def updateId(self, new_id):
                     '''
                     new_id = int(parentId) * 10 + (int(self.id) % 10)
                     new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
                     # If this proc unit has not inputs
                     #if self.inputId == '0':
                         #new_inputId = 0
                     n = 1
                     for opConfObj in self.opConfObjList:
                         idOp = str(int(new_id) * 10 + n)
                         opConfObj.updateId(idOp)
                         n += 1
                     self.parentId = str(parentId)
                     self.id = str(new_id)
                     #self.inputId = str(new_inputId)
                     '''
                     n = 1
                 def getInputId(self):
                     return self.inputId
                 def getOperationObjList(self):
                     return self.opConfObjList
                 def getOperationObj(self, name=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.name != name:
                             continue
                         return opConfObj
                     return None
                 def getOpObjfromParamValue(self, value=None):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
                             continue
                         return opConfObj
                     return None
                 def getProcUnitObj(self):
                     return self.procUnitObj
-                def setup(self, project_id, id, name, datatype, inputId):
+                def setup(self, project_id, id, name, datatype, inputId, err_queue):
                     '''
                         id sera el topico a publicar
                         inputId sera el topico a subscribirse
                     '''
                     # Compatible with old signal chain version
                     if datatype == None and name == None:
                         raise ValueError('datatype or name should be defined')
                     #Definir una condicion para inputId cuando sea 0
                     if name == None:
                         if 'Proc' in datatype:
                             name = datatype
                         else:
                             name = '%sProc' % (datatype)
                     if datatype == None:
                         datatype = name.replace('Proc', '')
                     self.id = str(id)
                     self.project_id = project_id
                     self.name = name
                     self.datatype = datatype
                     self.inputId = inputId
+                    self.err_queue = err_queue
                     self.opConfObjList = []
                     self.addOperation(name='run', optype='self')
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                     self.addOperation(name='run')
                 def addParameter(self, **kwargs):
                     '''
                     Add parameters to 'run' operation
                     '''
                     opObj = self.opConfObjList[0]
                     opObj.addParameter(**kwargs)
                     return opObj
                 def addOperation(self, name, optype='self'):
                     '''
                         Actualizacion - > proceso comunicacion
                         En el caso de optype='self', elminar. DEfinir comuncacion IPC -> Topic
                         definir el tipoc de socket o comunicacion ipc++
                     '''
                     id = self.__getNewId()
                     priority = self.__getPriority() # Sin mucho sentido, pero puede usarse
                     opConfObj = OperationConf()
-                    opConfObj.setup(id, name=name, priority=priority, type=optype, project_id=self.project_id)
+                    opConfObj.setup(id, name=name, priority=priority, type=optype, project_id=self.project_id, err_queue=self.err_queue)
                     self.opConfObjList.append(opConfObj)
                     return opConfObj
                 def makeXml(self, projectElement):
                     procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
                     procUnitElement.set('id', str(self.id))
                     procUnitElement.set('name', self.name)
                     procUnitElement.set('datatype', self.datatype)
                     procUnitElement.set('inputId', str(self.inputId))
                     for opConfObj in self.opConfObjList:
                         opConfObj.makeXml(procUnitElement)
                 def readXml(self, upElement, project_id):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.inputId = upElement.get('inputId')
                     self.project_id = str(project_id)
                     if self.ELEMENTNAME == 'ReadUnit':
                         self.datatype = self.datatype.replace('Reader', '')
                     if self.ELEMENTNAME == 'ProcUnit':
                         self.datatype = self.datatype.replace('Proc', '')
                     if self.inputId == 'None':
                         self.inputId = '0'
                     self.opConfObjList = []
                     opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement, project_id)
                         self.opConfObjList.append(opConfObj)
                 def printattr(self):
                     print('%s[%s]: name = %s, datatype = %s, inputId = %s, project_id = %s' % (self.ELEMENTNAME,
                                                                               self.id,
                                                                               self.name,
                                                                               self.datatype,
                                                                               self.inputId,
                                                                               self.project_id))
                     for opConfObj in self.opConfObjList:
                         opConfObj.printattr()
                 def getKwargs(self):
                     opObj = self.opConfObjList[0]
                     kwargs = opObj.getKwargs()
                     return kwargs
                 def createObjects(self):
                     '''
                     Instancia de unidades de procesamiento.
                     '''
                     className = eval(self.name)
                     kwargs = self.getKwargs()
-                    procUnitObj = className(self.id, self.inputId, self.project_id, **kwargs) # necesitan saber su id y su entrada por fines de ipc
+                    procUnitObj = className(self.id, self.inputId, self.project_id, self.err_queue, 'ProcUnit', **kwargs)
                     log.success('creating process...', self.name)
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self' and opConfObj.name == 'run':
                             continue
                         elif opConfObj.type == 'self':
                             opObj = getattr(procUnitObj, opConfObj.name)
                         else:
                             opObj = opConfObj.createObject()
-                        log.success('creating operation: {}, type:{}'.format(
+                        log.success('adding operation: {}, type:{}'.format(
                             opConfObj.name,
                             opConfObj.type), self.name)
                         procUnitObj.addOperation(opConfObj, opObj)
                     procUnitObj.start()
                     self.procUnitObj = procUnitObj
                 def close(self):
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self':
                             continue
                         opObj = self.procUnitObj.getOperationObj(opConfObj.id)
                         opObj.close()
                     self.procUnitObj.close()
                     return
             class ReadUnitConf(ProcUnitConf):
                 ELEMENTNAME = 'ReadUnit'
                 def __init__(self):
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.opConfObjList = []
                 def getElementName(self):
                     return self.ELEMENTNAME
-                def setup(self, project_id, id, name, datatype, path='', startDate='', endDate='',
+                def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
                           startTime='', endTime='', server=None, **kwargs):
                     '''
                     *****el id del proceso sera el Topico
                     Adicion de {topic}, si no esta presente -> error
                     kwargs deben ser trasmitidos en la instanciacion
                     '''
                     # Compatible with old signal chain version
                     if datatype == None and name == None:
                         raise ValueError('datatype or name should be defined')
                     if name == None:
                         if 'Reader' in datatype:
                             name = datatype
                             datatype = name.replace('Reader','')
                         else:
                             name = '{}Reader'.format(datatype)
                     if datatype == None:
                         if 'Reader' in name:
                             datatype = name.replace('Reader','')
                         else:
                             datatype = name
                             name = '{}Reader'.format(name)
                     self.id = id
                     self.project_id = project_id
                     self.name = name
                     self.datatype = datatype
                     if path != '':
                         self.path = os.path.abspath(path)
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.server = server
+                    self.err_queue = err_queue
                     self.addRunOperation(**kwargs)
                 def update(self, **kwargs):
                     if 'datatype' in kwargs:
                         datatype = kwargs.pop('datatype')
                         if 'Reader' in datatype:
                             self.name = datatype
                         else:
                             self.name = '%sReader' % (datatype)
                         self.datatype = self.name.replace('Reader', '')
                     attrs = ('path', 'startDate', 'endDate',
                              'startTime', 'endTime')
                     for attr in attrs:
                         if attr in kwargs:
                             setattr(self, attr, kwargs.pop(attr))
                     self.updateRunOperation(**kwargs)
                 def removeOperations(self):
                     for obj in self.opConfObjList:
                         del obj
                     self.opConfObjList = []
                 def addRunOperation(self, **kwargs):
                     opObj = self.addOperation(name='run', optype='self')
                     if self.server is None:
                         opObj.addParameter(
                             name='datatype', value=self.datatype, format='str')
                         opObj.addParameter(name='path', value=self.path, format='str')
                         opObj.addParameter(
                             name='startDate', value=self.startDate, format='date')
                         opObj.addParameter(
                             name='endDate', value=self.endDate, format='date')
                         opObj.addParameter(
                             name='startTime', value=self.startTime, format='time')
                         opObj.addParameter(
                             name='endTime', value=self.endTime, format='time')
                         for key, value in list(kwargs.items()):
                             opObj.addParameter(name=key, value=value,
                                                format=type(value).__name__)
                     else:
                         opObj.addParameter(name='server', value=self.server, format='str')
                     return opObj
                 def updateRunOperation(self, **kwargs):
                     opObj = self.getOperationObj(name='run')
                     opObj.removeParameters()
                     opObj.addParameter(name='datatype', value=self.datatype, format='str')
                     opObj.addParameter(name='path', value=self.path, format='str')
                     opObj.addParameter(
                         name='startDate', value=self.startDate, format='date')
                     opObj.addParameter(name='endDate', value=self.endDate, format='date')
                     opObj.addParameter(
                         name='startTime', value=self.startTime, format='time')
                     opObj.addParameter(name='endTime', value=self.endTime, format='time')
                     for key, value in list(kwargs.items()):
                         opObj.addParameter(name=key, value=value,
                                            format=type(value).__name__)
                     return opObj
                 def readXml(self, upElement, project_id):
                     self.id = upElement.get('id')
                     self.name = upElement.get('name')
                     self.datatype = upElement.get('datatype')
                     self.project_id = str(project_id)  #yong
                     if self.ELEMENTNAME == 'ReadUnit':
                         self.datatype = self.datatype.replace('Reader', '')
                     self.opConfObjList = []
                     opElementList = upElement.iter(OperationConf().getElementName())
                     for opElement in opElementList:
                         opConfObj = OperationConf()
                         opConfObj.readXml(opElement, project_id)
                         self.opConfObjList.append(opConfObj)
                         if opConfObj.name == 'run':
                             self.path = opConfObj.getParameterValue('path')
                             self.startDate = opConfObj.getParameterValue('startDate')
                             self.endDate = opConfObj.getParameterValue('endDate')
                             self.startTime = opConfObj.getParameterValue('startTime')
                             self.endTime = opConfObj.getParameterValue('endTime')
             class Project(Process):
                 ELEMENTNAME = 'Project'
                 def __init__(self):
                     Process.__init__(self)
                     self.id = None
                     self.filename = None
                     self.description = None
                     self.email = None
                     self.alarm = None
                     self.procUnitConfObjDict = {}
+                    self.err_queue = Queue()
                 def __getNewId(self):
                     idList = list(self.procUnitConfObjDict.keys())
                     id = int(self.id) * 10
                     while True:
                         id += 1
                         if str(id) in idList:
                             continue
                         break
                     return str(id)
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def getId(self):
                     return self.id
                 def updateId(self, new_id):
                     self.id = str(new_id)
                     keyList = list(self.procUnitConfObjDict.keys())
                     keyList.sort()
                     n = 1
                     newProcUnitConfObjDict = {}
                     for procKey in keyList:
                         procUnitConfObj = self.procUnitConfObjDict[procKey]
                         idProcUnit = str(int(self.id) * 10 + n)
                         procUnitConfObj.updateId(idProcUnit)
                         newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
                         n += 1
                     self.procUnitConfObjDict = newProcUnitConfObjDict
                 def setup(self, id=1, name='', description='', email=None, alarm=[]):
                     print(' ')
                     print('*' * 60)
                     print('*  Starting SIGNAL CHAIN PROCESSING (Multiprocessing) v%s *' % schainpy.__version__)
                     print('*' * 60)
                     print("*  Python " + python_version() + "  *")
                     print('*' * 19)
                     print(' ')
                     self.id = str(id)
                     self.description = description
                     self.email = email
                     self.alarm = alarm
+                    if name:
+                        self.name = '{} ({})'.format(Process.__name__, name)
                 def update(self, **kwargs):
                     for key, value in list(kwargs.items()):
                         setattr(self, key, value)
                 def clone(self):
                     p = Project()
                     p.procUnitConfObjDict = self.procUnitConfObjDict
                     return p
                 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
                     '''
                     Actualizacion:
                         Se agrego un nuevo argumento: topic -relativo a la forma de comunicar los procesos simultaneos
                         * El id del proceso sera el topico al que se deben subscribir los procUnits para recibir la informacion(data)
                     '''
                     if id is None:
                         idReadUnit = self.__getNewId()
                     else:
                         idReadUnit = str(id)
                     readUnitConfObj = ReadUnitConf()
-                    readUnitConfObj.setup(self.id, idReadUnit, name, datatype, **kwargs)
+                    readUnitConfObj.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
                     self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     return readUnitConfObj
                 def addProcUnit(self, inputId='0', datatype=None, name=None):
                     '''
                     Actualizacion:
                         Se agrego dos nuevos argumentos: topic_read (lee data de otro procUnit) y topic_write(escribe o envia data a otro procUnit)
                         Deberia reemplazar a "inputId"
                         ** A fin de mantener el inputID, este sera la representaacion del topicoal que deben subscribirse. El ID propio de la intancia
                         (proceso) sera el topico de la publicacion, todo sera asignado de manera dinamica.
                     '''
-                    idProcUnit = self.__getNewId() #Topico para subscripcion
+                    idProcUnit = self.__getNewId()
                     procUnitConfObj = ProcUnitConf()
-                    procUnitConfObj.setup(self.id, idProcUnit, name, datatype, inputId) #topic_read, topic_write,
+                    procUnitConfObj.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
                     self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     return procUnitConfObj
                 def removeProcUnit(self, id):
                     if id in list(self.procUnitConfObjDict.keys()):
                         self.procUnitConfObjDict.pop(id)
                 def getReadUnitId(self):
                     readUnitConfObj = self.getReadUnitObj()
                     return readUnitConfObj.id
                 def getReadUnitObj(self):
                     for obj in list(self.procUnitConfObjDict.values()):
                         if obj.getElementName() == 'ReadUnit':
                             return obj
                     return None
                 def getProcUnitObj(self, id=None, name=None):
                     if id != None:
                         return self.procUnitConfObjDict[id]
                     if name != None:
                         return self.getProcUnitObjByName(name)
                     return None
                 def getProcUnitObjByName(self, name):
                     for obj in list(self.procUnitConfObjDict.values()):
                         if obj.name == name:
                             return obj
                     return None
                 def procUnitItems(self):
                     return list(self.procUnitConfObjDict.items())
                 def makeXml(self):
                     projectElement = Element('Project')
                     projectElement.set('id', str(self.id))
                     projectElement.set('name', self.name)
                     projectElement.set('description', self.description)
                     for procUnitConfObj in list(self.procUnitConfObjDict.values()):
                         procUnitConfObj.makeXml(projectElement)
                     self.projectElement = projectElement
                 def writeXml(self, filename=None):
                     if filename == None:
                         if self.filename:
                             filename = self.filename
                         else:
                             filename = 'schain.xml'
                     if not filename:
                         print('filename has not been defined. Use setFilename(filename) for do it.')
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.access(os.path.dirname(abs_file), os.W_OK):
                         print('No write permission on %s' % os.path.dirname(abs_file))
                         return 0
                     if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
                         print('File %s already exists and it could not be overwriten' % abs_file)
                         return 0
                     self.makeXml()
                     ElementTree(self.projectElement).write(abs_file, method='xml')
                     self.filename = abs_file
                     return 1
                 def readXml(self, filename=None):
                     if not filename:
                         print('filename is not defined')
                         return 0
                     abs_file = os.path.abspath(filename)
                     if not os.path.isfile(abs_file):
                         print('%s file does not exist' % abs_file)
                         return 0
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     try:
                         self.projectElement = ElementTree().parse(abs_file)
                     except:
                         print('Error reading %s, verify file format' % filename)
                         return 0
                     self.project = self.projectElement.tag
                     self.id = self.projectElement.get('id')
                     self.name = self.projectElement.get('name')
                     self.description = self.projectElement.get('description')
                     readUnitElementList = self.projectElement.iter(
                         ReadUnitConf().getElementName())
                     for readUnitElement in readUnitElementList:
                         readUnitConfObj = ReadUnitConf()
                         readUnitConfObj.readXml(readUnitElement, self.id)
                         self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     procUnitElementList = self.projectElement.iter(
                         ProcUnitConf().getElementName())
                     for procUnitElement in procUnitElementList:
                         procUnitConfObj = ProcUnitConf()
                         procUnitConfObj.readXml(procUnitElement, self.id)
                         self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     self.filename = abs_file
                     return 1
                 def __str__(self):
-                    print('Project[%s]: name = %s, description = %s, project_id = %s' % (self.id,
+                    print('Project: name = %s, description = %s, id = %s' % (
                                                                         self.name,
                                                                         self.description,
-                                                                        self.project_id))
+                                                                        self.id))
                     for procUnitConfObj in self.procUnitConfObjDict.values():
                         print(procUnitConfObj)
                 def createObjects(self):
                     keys = list(self.procUnitConfObjDict.keys())
                     keys.sort()
                     for key in keys:
                         self.procUnitConfObjDict[key].createObjects()
-                def __handleError(self, procUnitConfObj, modes=None, stdout=True):
+                def monitor(self):
+                    t = Thread(target=self.__monitor, args=(self.err_queue, self.ctx))
+                    t.start()
+                def __monitor(self, queue, ctx):
                     import socket
-                    if modes is None:
+                    procs = 0
-                        modes = self.alarm
+                    err_msg = ''
+                    while True:
+                        msg = queue.get()
+                        if '#_start_#' in msg:
+                            procs += 1
+                        elif '#_end_#' in msg:
+                            procs -=1
+                        else:
+                            err_msg = msg
+                        if procs == 0 or 'Traceback' in err_msg:
+                            break
+                        time.sleep(0.1)
+                    if '|' in err_msg:
+                        name, err = err_msg.split('|')
+                        if 'SchainWarning' in err:
+                            log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
+                        elif 'SchainError' in err:
+                            log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
+                        else:
+                            log.error(err, name)
+                    else:
+                        name, err = self.name, err_msg
-                    if not self.alarm:
+                    time.sleep(2)
-                        modes = []
-                    err = traceback.format_exception(sys.exc_info()[0],
+                    for conf in self.procUnitConfObjDict.values():
-                                                     sys.exc_info()[1],
+                        for confop in conf.opConfObjList:
-                                                     sys.exc_info()[2])
+                            if confop.type == 'external':
+                                confop.opObj.terminate()
+                        conf.procUnitObj.terminate()
-                    log.error('{}'.format(err[-1]), procUnitConfObj.name)
+                    ctx.term()
                     message = ''.join(err)
-                    if stdout:
+                    if err_msg:
-                        sys.stderr.write(message)
                         subject = 'SChain v%s: Error running %s\n' % (
-                        schainpy.__version__, procUnitConfObj.name)
+                            schainpy.__version__, self.name)
-                    subtitle = '%s: %s\n' % (
+                        subtitle = 'Hostname: %s\n' % socket.gethostbyname(
-                        procUnitConfObj.getElementName(), procUnitConfObj.name)
-                    subtitle += 'Hostname: %s\n' % socket.gethostbyname(
                             socket.gethostname())
                         subtitle += 'Working directory: %s\n' % os.path.abspath('./')
                         subtitle += 'Configuration file: %s\n' % self.filename
                         subtitle += 'Time: %s\n' % str(datetime.datetime.now())
                         readUnitConfObj = self.getReadUnitObj()
                         if readUnitConfObj:
                             subtitle += '\nInput parameters:\n'
                             subtitle += '[Data path = %s]\n' % readUnitConfObj.path
                             subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
                             subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
                             subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
                             subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
                             subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
                         a = Alarm(
-                        modes=modes,
+                            modes=self.alarm,
                             email=self.email,
                             message=message,
                             subject=subject,
                             subtitle=subtitle,
                             filename=self.filename
                         )
-                    return a
+                        a.start()
                 def isPaused(self):
                     return 0
                 def isStopped(self):
                     return 0
                 def runController(self):
                     '''
                     returns 0 when this process has been stopped, 1 otherwise
                     '''
                     if self.isPaused():
                         print('Process suspended')
                         while True:
                             time.sleep(0.1)
                             if not self.isPaused():
                                 break
                             if self.isStopped():
                                 break
                         print('Process reinitialized')
                     if self.isStopped():
                         print('Process stopped')
                         return 0
                     return 1
                 def setFilename(self, filename):
                     self.filename = filename
-                def setProxyCom(self):
+                def setProxy(self):
                     if not os.path.exists('/tmp/schain'):
                         os.mkdir('/tmp/schain')
                     self.ctx = zmq.Context()
                     xpub = self.ctx.socket(zmq.XPUB)
                     xpub.bind('ipc:///tmp/schain/{}_pub'.format(self.id))
                     xsub = self.ctx.socket(zmq.XSUB)
                     xsub.bind('ipc:///tmp/schain/{}_sub'.format(self.id))
+                    self.monitor()
                     try:
                         zmq.proxy(xpub, xsub)
-                    except: # zmq.ContextTerminated:
+                    except zmq.ContextTerminated:
                         xpub.close()
                         xsub.close()
                 def run(self):
                     log.success('Starting {}: {}'.format(self.name, self.id), tag='')
                     self.start_time = time.time()
                     self.createObjects()
-                    # t = Thread(target=wait, args=(self.ctx, ))
+                    self.setProxy()
-                    # t.start()
+                    log.success('{} Done (Time: {}s)'.format(
-                    self.setProxyCom()
-                    # Iniciar todos los procesos .start(), monitoreo de procesos. ELiminar lo de abajo
-                    log.success('{} Done (time: {}s)'.format(
                         self.name,
-                        time.time()-self.start_time))
+                        time.time()-self.start_time), '')

schainpy/model/io/bltrIO_param.py +3 -3

             '''
             Created on Nov 9, 2016
             @author: roj- LouVD
             '''
             import os
             import sys
             import time
             import glob
             import datetime
             import numpy
+            import schainpy.admin
             from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator
             from schainpy.model.data.jrodata import Parameters
             from schainpy.model.io.jroIO_base import JRODataReader, isNumber
             from schainpy.utils import log
             FILE_HEADER_STRUCTURE = numpy.dtype([
                 ('FMN', '<u4'),
                 ('nrec', '<u4'),
                 ('fr_offset', '<u4'),
                 ('id', '<u4'),
                 ('site', 'u1', (32,))
             ])
             REC_HEADER_STRUCTURE = numpy.dtype([
                 ('rmn', '<u4'),
                 ('rcounter', '<u4'),
                 ('nr_offset', '<u4'),
                 ('tr_offset', '<u4'),
                 ('time', '<u4'),
                 ('time_msec', '<u4'),
                 ('tag', 'u1', (32,)),
                 ('comments', 'u1', (32,)),
                 ('lat', '<f4'),
                 ('lon', '<f4'),
                 ('gps_status', '<u4'),
                 ('freq', '<u4'),
                 ('freq0', '<u4'),
                 ('nchan', '<u4'),
                 ('delta_r', '<u4'),
                 ('nranges', '<u4'),
                 ('r0', '<u4'),
                 ('prf', '<u4'),
                 ('ncoh', '<u4'),
                 ('npoints', '<u4'),
                 ('polarization', '<i4'),
                 ('rx_filter', '<u4'),
                 ('nmodes', '<u4'),
                 ('dmode_index', '<u4'),
                 ('dmode_rngcorr', '<u4'),
                 ('nrxs', '<u4'),
                 ('acf_length', '<u4'),
                 ('acf_lags', '<u4'),
                 ('sea_to_atmos', '<f4'),
                 ('sea_notch', '<u4'),
                 ('lh_sea', '<u4'),
                 ('hh_sea', '<u4'),
                 ('nbins_sea', '<u4'),
                 ('min_snr', '<f4'),
                 ('min_cc', '<f4'),
                 ('max_time_diff', '<f4')
             ])
             DATA_STRUCTURE = numpy.dtype([
                 ('range', '<u4'),
                 ('status', '<u4'),
                 ('zonal', '<f4'),
                 ('meridional', '<f4'),
                 ('vertical', '<f4'),
                 ('zonal_a', '<f4'),
                 ('meridional_a', '<f4'),
                 ('corrected_fading', '<f4'),  # seconds
                 ('uncorrected_fading', '<f4'),  # seconds
                 ('time_diff', '<f4'),
                 ('major_axis', '<f4'),
                 ('axial_ratio', '<f4'),
                 ('orientation', '<f4'),
                 ('sea_power', '<u4'),
                 ('sea_algorithm', '<u4')
             ])
             @MPDecorator
             class BLTRParamReader(JRODataReader, ProcessingUnit):
                 '''
                 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR
                 from *.sswma files
                 '''
                 ext = '.sswma'
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                     self.counter_records = 0
                     self.flagNoMoreFiles = 0
                     self.isConfig = False
                     self.filename = None
                 def setup(self,
                           path=None,
                           startDate=None,
                           endDate=None,
                           ext=None,
                           startTime=datetime.time(0, 0, 0),
                           endTime=datetime.time(23, 59, 59),
                           timezone=0,
                           status_value=0,
                           **kwargs):
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.status_value = status_value
                     self.datatime = datetime.datetime(1900,1,1)
                     self.delay = kwargs.get('delay', 10)
                     self.online = kwargs.get('online', False)
                     self.nTries = kwargs.get('nTries', 3)
                     if self.path is None:
                         raise ValueError("The path is not valid")
                     if ext is None:
                         ext = self.ext
                     self.fileList = self.search_files(self.path, startDate, endDate, ext)
                     self.timezone = timezone
                     self.fileIndex = 0
                     if not self.fileList:
                         raise Warning("There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' " % (
                             path))
                     self.setNextFile()
                 def search_last_file(self):
                     '''
                     Get last file and add it to the list
                     '''
                     for n in range(self.nTries+1):
                         if n>0:
                             log.warning(
                                 "Waiting %0.2f seconds for the next file, try %03d ..." % (self.delay, n+1),
                                 self.name
                             )
                             time.sleep(self.delay)
                         file_list = os.listdir(self.path)
                         file_list.sort()
                         if file_list:
                             if self.filename:
                                 if file_list[-1] not in self.filename:
                                     return file_list[-1]
                                 else:
                                     continue
                             return file_list[-1]
                     return 0
                 def search_files(self, path, startDate, endDate, ext):
                     '''
                      Searching for BLTR rawdata file in path
                      Creating a list of file to proces included in [startDate,endDate]
                      Input:
                          path - Path to find BLTR rawdata files
                          startDate - Select file from this date
                          enDate - Select file until this date
                          ext - Extension of the file to read
                     '''
                     log.success('Searching files in {} '.format(path), 'BLTRParamReader')
                     foldercounter = 0
                     fileList0 = glob.glob1(path, "*%s" % ext)
                     fileList0.sort()
                     for thisFile in fileList0:
                         year = thisFile[-14:-10]
                         if not isNumber(year):
                             continue
                         month = thisFile[-10:-8]
                         if not isNumber(month):
                             continue
                         day = thisFile[-8:-6]
                         if not isNumber(day):
                             continue
                         year, month, day = int(year), int(month), int(day)
                         dateFile = datetime.date(year, month, day)
                         if (startDate > dateFile) or (endDate < dateFile):
                             continue
                         yield thisFile
                     return
                 def setNextFile(self):
                     if self.online:
                         filename = self.search_last_file()
                         if not filename:
                             self.flagNoMoreFiles = 1
                             return 0
                     else:
                         try:
                             filename = next(self.fileList)
                         except StopIteration:
                             self.flagNoMoreFiles = 1
                             return 0
                     log.success('Opening {}'.format(filename), 'BLTRParamReader')
                     dirname, name = os.path.split(filename)
                     # 'peru2' ---> Piura  -   'peru1' ---> Huancayo or Porcuya
                     self.siteFile = filename.split('.')[0]
                     if self.filename is not None:
                         self.fp.close()
                     self.filename = os.path.join(self.path, filename)
                     self.fp = open(self.filename, 'rb')
                     self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
                     self.nrecords = self.header_file['nrec'][0]
                     self.sizeOfFile = os.path.getsize(self.filename)
                     self.counter_records = 0
                     self.flagIsNewFile = 0
                     self.fileIndex += 1
                     return 1
                 def readNextBlock(self):
                     while True:
                         if not self.online and self.counter_records == self.nrecords:
                             self.flagIsNewFile = 1
                             if not self.setNextFile():
                                 return 0
                         try:
                             pointer = self.fp.tell()
                             self.readBlock()
                         except:
                             if self.online and self.waitDataBlock(pointer, 38512) == 1:
                                 continue
                             else:
                                 if not self.setNextFile():
                                     return 0
                         if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
                            (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
                             log.warning(
                                 'Reading Record No. {}/{} -> {} [Skipping]'.format(
                                     self.counter_records,
                                     self.nrecords,
                                     self.datatime.ctime()),
                                 'BLTRParamReader')
                             continue
                         break
                     log.log('Reading Record No. {} -> {}'.format(
                         self.counter_records,
                         # self.nrecords,
                         self.datatime.ctime()), 'BLTRParamReader')
                     return 1
                 def readBlock(self):
                     pointer = self.fp.tell()
                     header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
                     self.nchannels = int(header_rec['nchan'][0] / 2)
                     self.kchan = header_rec['nrxs'][0]
                     self.nmodes = header_rec['nmodes'][0]
                     self.nranges = header_rec['nranges'][0]
                     self.fp.seek(pointer)
                     self.height = numpy.empty((self.nmodes, self.nranges))
                     self.snr = numpy.empty((self.nmodes, int(self.nchannels), self.nranges))
                     self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
                     self.flagDiscontinuousBlock = 0
                     for mode in range(self.nmodes):
                         self.readHeader()
                         data = self.readData()
                         self.height[mode] = (data[0] - self.correction) / 1000.
                         self.buffer[mode] = data[1]
                         self.snr[mode] = data[2]
                     self.counter_records = self.counter_records + self.nmodes
                     return
                 def readHeader(self):
                     '''
                     RecordHeader of BLTR rawdata file
                     '''
                     header_structure = numpy.dtype(
                         REC_HEADER_STRUCTURE.descr + [
                             ('antenna_coord', 'f4', (2, int(self.nchannels))),
                             ('rx_gains', 'u4', (int(self.nchannels),)),
                             ('rx_analysis', 'u4', (int(self.nchannels),))
                         ]
                     )
                     self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
                     self.lat = self.header_rec['lat'][0]
                     self.lon = self.header_rec['lon'][0]
                     self.delta = self.header_rec['delta_r'][0]
                     self.correction = self.header_rec['dmode_rngcorr'][0]
                     self.imode = self.header_rec['dmode_index'][0]
                     self.antenna = self.header_rec['antenna_coord']
                     self.rx_gains = self.header_rec['rx_gains']
                     self.time = self.header_rec['time'][0]
                     dt = datetime.datetime.utcfromtimestamp(self.time)
                     if dt.date()>self.datatime.date():
                         self.flagDiscontinuousBlock = 1
                     self.datatime = dt
                 def readData(self):
                     '''
                     Reading and filtering data block record of BLTR rawdata file,
                     filtering is according to status_value.
                     Input:
                         status_value - Array data is set to NAN for values that are not
                         equal to status_value
                     '''
                     self.nchannels = int(self.nchannels)
                     data_structure = numpy.dtype(
                         DATA_STRUCTURE.descr + [
                             ('rx_saturation', 'u4', (self.nchannels,)),
                             ('chan_offset', 'u4', (2 * self.nchannels,)),
                             ('rx_amp', 'u4', (self.nchannels,)),
                             ('rx_snr', 'f4', (self.nchannels,)),
                             ('cross_snr', 'f4', (self.kchan,)),
                             ('sea_power_relative', 'f4', (self.kchan,))]
                     )
                     data = numpy.fromfile(self.fp, data_structure, self.nranges)
                     height = data['range']
                     winds = numpy.array(
                         (data['zonal'], data['meridional'], data['vertical']))
                     snr = data['rx_snr'].T
                     winds[numpy.where(winds == -9999.)] = numpy.nan
                     winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
                     snr[numpy.where(snr == -9999.)] = numpy.nan
                     snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
                     snr = numpy.power(10, snr / 10)
                     return height, winds, snr
                 def set_output(self):
                     '''
                     Storing data from databuffer to dataOut object
                     '''
                     self.dataOut.data_SNR = self.snr
                     self.dataOut.height = self.height
                     self.dataOut.data = self.buffer
                     self.dataOut.utctimeInit = self.time
                     self.dataOut.utctime = self.dataOut.utctimeInit
                     self.dataOut.useLocalTime = False
                     self.dataOut.paramInterval = 157
                     self.dataOut.timezone = self.timezone
                     self.dataOut.site = self.siteFile
                     self.dataOut.nrecords = self.nrecords / self.nmodes
                     self.dataOut.sizeOfFile = self.sizeOfFile
                     self.dataOut.lat = self.lat
                     self.dataOut.lon = self.lon
                     self.dataOut.channelList = list(range(self.nchannels))
                     self.dataOut.kchan = self.kchan
                     self.dataOut.delta = self.delta
                     self.dataOut.correction = self.correction
                     self.dataOut.nmodes = self.nmodes
                     self.dataOut.imode = self.imode
                     self.dataOut.antenna = self.antenna
                     self.dataOut.rx_gains = self.rx_gains
                     self.dataOut.flagNoData = False
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                 def getData(self):
                     '''
                     Storing data from databuffer to dataOut object
                     '''
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
-                        self.dataOut.error = 'No More files to read'
+                        raise schainpy.admin.SchainError('No More files to read')
-                        return
                     if not self.readNextBlock():
                         self.dataOut.flagNoData = True
-                        self.dataOut.error = 'Time for wait new file reach!!!'
+                        raise schainpy.admin.SchainError('Time for wait new file reach!!!')
                     self.set_output()
                     return 1
   No newline at end of file

schainpy/model/io/bltrIO_spectra.py +2 -6

             import os
             import sys
             import glob
             import fnmatch
             import datetime
             import time
             import re
             import h5py
             import numpy
             import pylab as plb
             from scipy.optimize import curve_fit
             from scipy import asarray as ar, exp
             SPEED_OF_LIGHT = 299792458
             SPEED_OF_LIGHT = 3e8
-            try:
-                from gevent import sleep
-            except:
-                from time import sleep
             from .utils import folder_in_range
+            import schainpy.admin
             from schainpy.model.data.jrodata import Spectra
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.utils import log
             from schainpy.model.io.jroIO_base import JRODataReader
             def pol2cart(rho, phi):
                 x = rho * numpy.cos(phi)
                 y = rho * numpy.sin(phi)
                 return(x, y)
             FILE_STRUCTURE = numpy.dtype([  # HEADER 48bytes
                 ('FileMgcNumber', '<u4'),  # 0x23020100
                 ('nFDTdataRecors', '<u4'),
                 ('OffsetStartHeader', '<u4'),
                 ('RadarUnitId', '<u4'),
                 ('SiteName', 'S32'),  # Null terminated
             ])
             class FileHeaderBLTR():
                 def __init__(self, fo):
                     self.fo = fo
                     self.size = 48
                     self.read()
                 def read(self):
                     header = numpy.fromfile(self.fo, FILE_STRUCTURE, 1)
                     self.FileMgcNumber = hex(header['FileMgcNumber'][0])
                     self.nFDTdataRecors = int(header['nFDTdataRecors'][0])
                     self.RadarUnitId = int(header['RadarUnitId'][0])
                     self.OffsetStartHeader = int(header['OffsetStartHeader'][0])
                     self.SiteName = header['SiteName'][0]
                 def write(self, fp):
                     headerTuple = (self.FileMgcNumber,
                                    self.nFDTdataRecors,
                                    self.RadarUnitId,
                                    self.SiteName,
                                    self.size)
                     header = numpy.array(headerTuple, FILE_STRUCTURE)
                     header.tofile(fp)
                     ''' ndarray.tofile(fid, sep, format)    Write array to a file as text or binary (default).
                     fid : file or str
                     An open file object, or a string containing a filename.
                     sep : str
                     Separator between array items for text output. If "" (empty), a binary file is written,
                     equivalent to file.write(a.tobytes()).
                     format : str
                     Format string for text file output. Each entry in the array is formatted to text by
                     first converting it to the closest Python type, and then using "format" % item.
                     '''
                     return 1
             RECORD_STRUCTURE = numpy.dtype([  # RECORD HEADER 180+20N bytes
                 ('RecMgcNumber', '<u4'),  # 0x23030001
                 ('RecCounter', '<u4'),  # Record counter(0,1, ...)
                 # Offset to start of next record form start of this record
                                         ('Off2StartNxtRec', '<u4'),
                                         # Offset to start of data from start of this record
                                         ('Off2StartData', '<u4'),
                                         # Epoch time stamp of start of acquisition (seconds)
                                         ('nUtime', '<i4'),
                                         # Millisecond component of time stamp (0,...,999)
                                         ('nMilisec', '<u4'),
                                         # Experiment tag name (null terminated)
                                         ('ExpTagName', 'S32'),
                                         # Experiment comment (null terminated)
                                         ('ExpComment', 'S32'),
                                         # Site latitude (from GPS) in degrees (positive implies North)
                                         ('SiteLatDegrees', '<f4'),
                                         # Site longitude (from GPS) in degrees (positive implies East)
                                         ('SiteLongDegrees', '<f4'),
                                         # RTC GPS engine status (0=SEEK, 1=LOCK, 2=NOT FITTED, 3=UNAVAILABLE)
                                         ('RTCgpsStatus', '<u4'),
                                         ('TransmitFrec', '<u4'),  # Transmit frequency (Hz)
                                         ('ReceiveFrec', '<u4'),  # Receive frequency
                                         # First local oscillator frequency (Hz)
                                         ('FirstOsciFrec', '<u4'),
                                         # (0="O", 1="E", 2="linear 1", 3="linear2")
                                         ('Polarisation', '<u4'),
                                         # Receiver filter settings (0,1,2,3)
                                         ('ReceiverFiltSett', '<u4'),
                                         # Number of modes in use (1 or 2)
                                         ('nModesInUse', '<u4'),
                                         # Dual Mode index number for these data (0 or 1)
                                         ('DualModeIndex', '<u4'),
                                         # Dual Mode range correction for these data (m)
                                         ('DualModeRange', '<u4'),
                                         # Number of digital channels acquired (2*N)
                                         ('nDigChannels', '<u4'),
                                         # Sampling resolution (meters)
                                         ('SampResolution', '<u4'),
                                         # Number of range gates sampled
                                         ('nHeights', '<u4'),
                                         # Start range of sampling (meters)
                                         ('StartRangeSamp', '<u4'),
                                         ('PRFhz', '<u4'),  # PRF (Hz)
                                         ('nCohInt', '<u4'),  # Integrations
                                         # Number of data points transformed
                                         ('nProfiles', '<u4'),
                                         # Number of receive beams stored in file (1 or N)
                                         ('nChannels', '<u4'),
                                         ('nIncohInt', '<u4'),  # Number of spectral averages
                                         # FFT windowing index (0 = no window)
                                         ('FFTwindowingInd', '<u4'),
                                         # Beam steer angle (azimuth) in degrees (clockwise from true North)
                                         ('BeamAngleAzim', '<f4'),
                                         # Beam steer angle (zenith) in degrees (0=> vertical)
                                         ('BeamAngleZen', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaCoord0', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaAngl0', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaCoord1', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaAngl1', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaCoord2', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaAngl2', '<f4'),
                                         # Receiver phase calibration (degrees) - N values
                                         ('RecPhaseCalibr0', '<f4'),
                                         # Receiver phase calibration (degrees) - N values
                                         ('RecPhaseCalibr1', '<f4'),
                                         # Receiver phase calibration (degrees) - N values
                                         ('RecPhaseCalibr2', '<f4'),
                                         # Receiver amplitude calibration (ratio relative to receiver one) - N values
                                         ('RecAmpCalibr0', '<f4'),
                                         # Receiver amplitude calibration (ratio relative to receiver one) - N values
                                         ('RecAmpCalibr1', '<f4'),
                                         # Receiver amplitude calibration (ratio relative to receiver one) - N values
                                         ('RecAmpCalibr2', '<f4'),
                                         # Receiver gains in dB - N values
                                         ('ReceiverGaindB0', '<i4'),
                                         # Receiver gains in dB - N values
                                         ('ReceiverGaindB1', '<i4'),
                                         # Receiver gains in dB - N values
                                         ('ReceiverGaindB2', '<i4'),
             ])
             class RecordHeaderBLTR():
                 def __init__(self, fo):
                     self.fo = fo
                     self.OffsetStartHeader = 48
                     self.Off2StartNxtRec = 811248
                 def read(self, block):
                     OffRHeader = self.OffsetStartHeader + block * self.Off2StartNxtRec
                     self.fo.seek(OffRHeader, os.SEEK_SET)
                     header = numpy.fromfile(self.fo, RECORD_STRUCTURE, 1)
                     self.RecMgcNumber = hex(header['RecMgcNumber'][0])  # 0x23030001
                     self.RecCounter = int(header['RecCounter'][0])
                     self.Off2StartNxtRec = int(header['Off2StartNxtRec'][0])
                     self.Off2StartData = int(header['Off2StartData'][0])
                     self.nUtime = header['nUtime'][0]
                     self.nMilisec = header['nMilisec'][0]
                     self.ExpTagName = '' # str(header['ExpTagName'][0])
                     self.ExpComment = '' # str(header['ExpComment'][0])
                     self.SiteLatDegrees = header['SiteLatDegrees'][0]
                     self.SiteLongDegrees = header['SiteLongDegrees'][0]
                     self.RTCgpsStatus = header['RTCgpsStatus'][0]
                     self.TransmitFrec = header['TransmitFrec'][0]
                     self.ReceiveFrec = header['ReceiveFrec'][0]
                     self.FirstOsciFrec = header['FirstOsciFrec'][0]
                     self.Polarisation = header['Polarisation'][0]
                     self.ReceiverFiltSett = header['ReceiverFiltSett'][0]
                     self.nModesInUse = header['nModesInUse'][0]
                     self.DualModeIndex = header['DualModeIndex'][0]
                     self.DualModeRange = header['DualModeRange'][0]
                     self.nDigChannels = header['nDigChannels'][0]
                     self.SampResolution = header['SampResolution'][0]
                     self.nHeights = header['nHeights'][0]
                     self.StartRangeSamp = header['StartRangeSamp'][0]
                     self.PRFhz = header['PRFhz'][0]
                     self.nCohInt = header['nCohInt'][0]
                     self.nProfiles = header['nProfiles'][0]
                     self.nChannels = header['nChannels'][0]
                     self.nIncohInt = header['nIncohInt'][0]
                     self.FFTwindowingInd = header['FFTwindowingInd'][0]
                     self.BeamAngleAzim = header['BeamAngleAzim'][0]
                     self.BeamAngleZen = header['BeamAngleZen'][0]
                     self.AntennaCoord0 = header['AntennaCoord0'][0]
                     self.AntennaAngl0 = header['AntennaAngl0'][0]
                     self.AntennaCoord1 = header['AntennaCoord1'][0]
                     self.AntennaAngl1 = header['AntennaAngl1'][0]
                     self.AntennaCoord2 = header['AntennaCoord2'][0]
                     self.AntennaAngl2 = header['AntennaAngl2'][0]
                     self.RecPhaseCalibr0 = header['RecPhaseCalibr0'][0]
                     self.RecPhaseCalibr1 = header['RecPhaseCalibr1'][0]
                     self.RecPhaseCalibr2 = header['RecPhaseCalibr2'][0]
                     self.RecAmpCalibr0 = header['RecAmpCalibr0'][0]
                     self.RecAmpCalibr1 = header['RecAmpCalibr1'][0]
                     self.RecAmpCalibr2 = header['RecAmpCalibr2'][0]
                     self.ReceiverGaindB0 = header['ReceiverGaindB0'][0]
                     self.ReceiverGaindB1 = header['ReceiverGaindB1'][0]
                     self.ReceiverGaindB2 = header['ReceiverGaindB2'][0]
                     self.ipp = 0.5 * (SPEED_OF_LIGHT / self.PRFhz)
                     self.RHsize = 180 + 20 * self.nChannels
                     self.Datasize = self.nProfiles * self.nChannels * self.nHeights * 2 * 4
                     endFp = self.OffsetStartHeader + self.RecCounter * self.Off2StartNxtRec
                     if OffRHeader > endFp:
                         sys.stderr.write(
                             "Warning %s: Size value read from System Header is lower than it has to be\n" % fp)
                         return 0
                     if OffRHeader < endFp:
                         sys.stderr.write(
                             "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp)
                         return 0
                     return 1
             @MPDecorator
             class BLTRSpectraReader (ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.ext = ".fdt"
                     self.optchar = "P"
                     self.fpFile = None
                     self.fp = None
                     self.BlockCounter = 0
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.fileSelector = 0
                     self.Off2StartNxtRec = 0
                     self.RecCounter = 0
                     self.flagNoMoreFiles = 0
                     self.data_spc = None
                     self.data_cspc = None
                     self.path = None
                     self.OffsetStartHeader = 0
                     self.Off2StartData = 0
                     self.ipp = 0
                     self.nFDTdataRecors = 0
                     self.blocksize = 0
                     self.dataOut = Spectra()
                     self.dataOut.flagNoData = False
                 def search_files(self):
                     '''
                     Function that indicates the number of .fdt files that exist in the folder to be read.
                     It also creates an organized list with the names of the files to read.
                     '''
                     files = glob.glob(os.path.join(self.path, '*{}'.format(self.ext)))
                     files = sorted(files)
                     for f in files:
                         filename = f.split('/')[-1]
                         if folder_in_range(filename.split('.')[1], self.startDate, self.endDate, '%Y%m%d'):
                             self.filenameList.append(f)
                 def run(self, **kwargs):
                     '''
                     This method will be the one that will initiate the data entry, will be called constantly.
                     You should first verify that your Setup () is set up and then continue to acquire
                     the data to be processed with getData ().
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
                 def setup(self,
                           path=None,
                           startDate=None,
                           endDate=None,
                           startTime=None,
                           endTime=None,
                           walk=True,
                           code=None,
                           online=False,
                           mode=None,
                           **kwargs):
                     self.isConfig = True
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.walk = walk
                     self.mode = int(mode)
                     self.search_files()
                     if self.filenameList:
                         self.readFile()
                 def getData(self):
                     '''
                     Before starting this function, you should check that there is still an unread file,
                     If there are still blocks to read or if the data block is empty.
                     You should call the file "read".
                     '''
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
-                        self.dataOut.error = 'No more files'
+                        raise schainpy.admin.SchainError('No more files')
                     self.readBlock()
                 def readFile(self):
                     '''
                     You must indicate if you are reading in Online or Offline mode and load the
                     The parameters for this file reading mode.
                     Then you must do 2 actions:
 . Get the BLTR FileHeader.
 . Start reading the first block.
                     '''
                     if self.fileSelector < len(self.filenameList):
                         log.success('Opening file: {}'.format(self.filenameList[self.fileSelector]), self.name)
                         self.fp = open(self.filenameList[self.fileSelector])
                         self.fheader = FileHeaderBLTR(self.fp)
                         self.rheader = RecordHeaderBLTR(self.fp)
                         self.nFDTdataRecors = self.fheader.nFDTdataRecors
                         self.fileSelector += 1
                         self.BlockCounter = 0
                         return 1
                     else:
                         self.flagNoMoreFiles = True
                         self.dataOut.flagNoData = True
                         return 0
                 def readBlock(self):
                     '''
                     It should be checked if the block has data, if it is not passed to the next file.
                     Then the following is done:
 . Read the RecordHeader
 . Fill the buffer with the current block number.
                     '''
                     if self.BlockCounter == self.nFDTdataRecors:
                         if not self.readFile():
                             return
                     if self.mode == 1:
                         self.rheader.read(self.BlockCounter+1)
                     elif self.mode == 0:
                         self.rheader.read(self.BlockCounter)
                     self.RecCounter = self.rheader.RecCounter
                     self.OffsetStartHeader = self.rheader.OffsetStartHeader
                     self.Off2StartNxtRec = self.rheader.Off2StartNxtRec
                     self.Off2StartData = self.rheader.Off2StartData
                     self.nProfiles = self.rheader.nProfiles
                     self.nChannels = self.rheader.nChannels
                     self.nHeights = self.rheader.nHeights
                     self.frequency = self.rheader.TransmitFrec
                     self.DualModeIndex = self.rheader.DualModeIndex
                     self.pairsList = [(0, 1), (0, 2), (1, 2)]
                     self.dataOut.pairsList = self.pairsList
                     self.nRdPairs = len(self.dataOut.pairsList)
                     self.dataOut.nRdPairs = self.nRdPairs
                     self.dataOut.heightList = (self.rheader.StartRangeSamp + numpy.arange(self.nHeights) * self.rheader.SampResolution) / 1000.
                     self.dataOut.channelList = range(self.nChannels)
                     self.dataOut.nProfiles=self.rheader.nProfiles
                     self.dataOut.nIncohInt=self.rheader.nIncohInt
                     self.dataOut.nCohInt=self.rheader.nCohInt
                     self.dataOut.ippSeconds= 1/float(self.rheader.PRFhz)
                     self.dataOut.PRF=self.rheader.PRFhz
                     self.dataOut.nFFTPoints=self.rheader.nProfiles
                     self.dataOut.utctime = self.rheader.nUtime + self.rheader.nMilisec/1000.
                     self.dataOut.timeZone = 0
                     self.dataOut.useLocalTime = False
                     self.dataOut.nmodes = 2
                     log.log('Reading block {} - {}'.format(self.BlockCounter, self.dataOut.datatime), self.name)
                     OffDATA = self.OffsetStartHeader + self.RecCounter * \
                         self.Off2StartNxtRec + self.Off2StartData
                     self.fp.seek(OffDATA, os.SEEK_SET)
                     self.data_fft = numpy.fromfile(self.fp, [('complex','<c8')], self.nProfiles*self.nChannels*self.nHeights )
                     self.data_fft = self.data_fft.astype(numpy.dtype('complex'))
                     self.data_block = numpy.reshape(self.data_fft,(self.nHeights, self.nChannels, self.nProfiles))
                     self.data_block = numpy.transpose(self.data_block, (1,2,0))
                     copy = self.data_block.copy()
                     spc = copy * numpy.conjugate(copy)
                     self.data_spc = numpy.absolute(spc)  # valor absoluto o magnitud
                     self.dataOut.data_spc = self.data_spc
                     cspc = self.data_block.copy()
                     self.data_cspc = self.data_block.copy()
                     for i in range(self.nRdPairs):
                         chan_index0 = self.dataOut.pairsList[i][0]
                         chan_index1 = self.dataOut.pairsList[i][1]
                         self.data_cspc[i, :, :] = cspc[chan_index0, :, :] * numpy.conjugate(cspc[chan_index1, :, :])
                     '''Getting Eij and Nij'''
                     (AntennaX0, AntennaY0) = pol2cart(
                         self.rheader.AntennaCoord0, self.rheader.AntennaAngl0 * numpy.pi / 180)
                     (AntennaX1, AntennaY1) = pol2cart(
                         self.rheader.AntennaCoord1, self.rheader.AntennaAngl1 * numpy.pi / 180)
                     (AntennaX2, AntennaY2) = pol2cart(
                         self.rheader.AntennaCoord2, self.rheader.AntennaAngl2 * numpy.pi / 180)
                     E01 = AntennaX0 - AntennaX1
                     N01 = AntennaY0 - AntennaY1
                     E02 = AntennaX0 - AntennaX2
                     N02 = AntennaY0 - AntennaY2
                     E12 = AntennaX1 - AntennaX2
                     N12 = AntennaY1 - AntennaY2
                     self.ChanDist = numpy.array(
                         [[E01, N01], [E02, N02], [E12, N12]])
                     self.dataOut.ChanDist = self.ChanDist
                     self.BlockCounter += 2
                     self.dataOut.data_spc = self.data_spc
                     self.dataOut.data_cspc =self.data_cspc

schainpy/model/io/jroIO_amisr.py 0 -1

             '''
             @author: Daniel Suarez
             '''
             import os
             import sys
             import glob
             import fnmatch
             import datetime
             import time
             import re
             import h5py
             import numpy
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             from schainpy.model.data.jroamisr import AMISR
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             class RadacHeader():
                 def __init__(self, fp):
                     header = 'Raw11/Data/RadacHeader'
                     self.beamCodeByPulse = fp.get(header+'/BeamCode')
                     self.beamCode = fp.get('Raw11/Data/Beamcodes')
                     self.code = fp.get(header+'/Code')
                     self.frameCount = fp.get(header+'/FrameCount')
                     self.modeGroup = fp.get(header+'/ModeGroup')
                     self.nsamplesPulse = fp.get(header+'/NSamplesPulse')
                     self.pulseCount = fp.get(header+'/PulseCount')
                     self.radacTime = fp.get(header+'/RadacTime')
                     self.timeCount = fp.get(header+'/TimeCount')
                     self.timeStatus = fp.get(header+'/TimeStatus')
                     self.nrecords = self.pulseCount.shape[0] #nblocks
                     self.npulses = self.pulseCount.shape[1] #nprofile
                     self.nsamples = self.nsamplesPulse[0,0] #ngates
                     self.nbeams = self.beamCode.shape[1]
                 def getIndexRangeToPulse(self, idrecord=0):
                     #indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
                     #startPulseCountId = indexToZero[0][0]
                     #endPulseCountId = startPulseCountId - 1
                     #range1 = numpy.arange(startPulseCountId,self.npulses,1)
                     #range2 = numpy.arange(0,startPulseCountId,1)
                     #return range1, range2
                     zero = 0
                     npulse = max(self.pulseCount[0,:]+1)-1
                     looking_index = numpy.where(self.pulseCount.value[idrecord,:]==npulse)[0]
                     getLastIndex = looking_index[-1]
                     index_data = numpy.arange(0,getLastIndex+1,1)
                     index_buffer = numpy.arange(getLastIndex+1,self.npulses,1)
                     return index_data, index_buffer
             class AMISRReader(ProcessingUnit):
                 path = None
                 startDate = None
                 endDate = None
                 startTime = None
                 endTime = None
                 walk = None
                 isConfig = False
                 def __init__(self):
                     self.set = None
                     self.subset = None
                     self.extension_file = '.h5'
                     self.dtc_str = 'dtc'
                     self.dtc_id = 0
                     self.status = True
                     self.isConfig = False
                     self.dirnameList = []
                     self.filenameList = []
                     self.fileIndex = None
                     self.flagNoMoreFiles = False
                     self.flagIsNewFile = 0
                     self.filename = ''
                     self.amisrFilePointer = None
                     self.radacHeaderObj = None
                     self.dataOut = self.__createObjByDefault()
                     self.datablock = None
                     self.rest_datablock = None
                     self.range = None
                     self.idrecord_count = 0
                     self.profileIndex = 0
                     self.index_amisr_sample = None
                     self.index_amisr_buffer = None
                     self.beamCodeByFrame = None
                     self.radacTimeByFrame = None
                     #atributos originales tal y como esta en el archivo de datos
                     self.beamCodesFromFile = None
                     self.radacTimeFromFile = None
                     self.rangeFromFile = None
                     self.dataByFrame = None
                     self.dataset = None
                     self.beamCodeDict = {}
                     self.beamRangeDict = {}
                     #experiment cgf file
                     self.npulsesint_fromfile = None
                     self.recordsperfile_fromfile = None
                     self.nbeamcodes_fromfile = None
                     self.ngates_fromfile = None
                     self.ippSeconds_fromfile = None
                     self.frequency_h5file = None
                     self.__firstFile = True
                     self.buffer_radactime = None
                     self.index4_schain_datablock = None
                     self.index4_buffer = None
                     self.schain_datablock = None
                     self.buffer = None
                     self.linear_pulseCount = None
                     self.npulseByFrame = None
                     self.profileIndex_offset = None
                     self.timezone = 'ut'
                     self.__waitForNewFile = 20
                     self.__filename_online = None
                 def __createObjByDefault(self):
                     dataObj = AMISR()
                     return dataObj
                 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.walk = walk
                 def __checkPath(self):
                     if os.path.exists(self.path):
                         self.status = 1
                     else:
                         self.status = 0
                         print('Path:%s does not exists'%self.path)
                     return
                 def __selDates(self, amisr_dirname_format):
                     try:
                         year = int(amisr_dirname_format[0:4])
                         month = int(amisr_dirname_format[4:6])
                         dom = int(amisr_dirname_format[6:8])
                         thisDate = datetime.date(year,month,dom)
                         if (thisDate>=self.startDate and thisDate <= self.endDate):
                             return amisr_dirname_format
                     except:
                         return None
                 def __findDataForDates(self,online=False):
                     if not(self.status):
                         return None
                     pat = '\d+.\d+'
                     dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
                     dirnameList = [x for x in dirnameList if x!=None]
                     dirnameList = [x.string for x in dirnameList]
                     if not(online):
                         dirnameList = [self.__selDates(x) for x in dirnameList]
                         dirnameList = [x for x in dirnameList if x!=None]
                     if len(dirnameList)>0:
                         self.status = 1
                         self.dirnameList = dirnameList
                         self.dirnameList.sort()
                     else:
                         self.status = 0
                         return None
                 def __getTimeFromData(self):
                     startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
                     endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                     print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
                     print('........................................')
                     filter_filenameList = []
                     self.filenameList.sort()
                     for i in range(len(self.filenameList)-1):
                         filename = self.filenameList[i]
                         fp = h5py.File(filename,'r')
                         time_str = fp.get('Time/RadacTimeString')
                         startDateTimeStr_File = time_str[0][0].split('.')[0]
                         junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                         startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                         endDateTimeStr_File = time_str[-1][-1].split('.')[0]
                         junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                         endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                         fp.close()
                         if self.timezone == 'lt':
                             startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
                             endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
                         if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
                             #self.filenameList.remove(filename)
                             filter_filenameList.append(filename)
                     filter_filenameList.sort()
                     self.filenameList = filter_filenameList
                     return 1
                 def __filterByGlob1(self, dirName):
                     filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
                     filterDict = {}
                     filterDict.setdefault(dirName)
                     filterDict[dirName] = filter_files
                     return filterDict
                 def __getFilenameList(self, fileListInKeys, dirList):
                     for value in fileListInKeys:
                         dirName = list(value.keys())[0]
                         for file in value[dirName]:
                             filename = os.path.join(dirName, file)
                             self.filenameList.append(filename)
                 def __selectDataForTimes(self, online=False):
                     #aun no esta implementado el filtro for tiempo
                     if not(self.status):
                         return None
                     dirList = [os.path.join(self.path,x) for x in self.dirnameList]
                     fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
                     self.__getFilenameList(fileListInKeys, dirList)
                     if not(online):
                         #filtro por tiempo
                         if not(self.all):
                             self.__getTimeFromData()
                         if len(self.filenameList)>0:
                             self.status = 1
                             self.filenameList.sort()
                         else:
                             self.status = 0
                             return None
                     else:
                         #get the last file - 1
                         self.filenameList = [self.filenameList[-2]]
                     new_dirnameList = []
                     for dirname in self.dirnameList:
                         junk = numpy.array([dirname in x for x in self.filenameList])
                         junk_sum = junk.sum()
                         if junk_sum > 0:
                             new_dirnameList.append(dirname)
                     self.dirnameList = new_dirnameList
                     return 1
                 def searchFilesOnLine(self,
                                         path,
                                         walk=True):
                     startDate = datetime.datetime.utcnow().date()
                     endDate = datetime.datetime.utcnow().date()
                     self.__setParameters(path=path, startDate=startDate, endDate=endDate, walk=walk)
                     self.__checkPath()
                     self.__findDataForDates(online=True)
                     self.dirnameList = [self.dirnameList[-1]]
                     self.__selectDataForTimes(online=True)
                     return
                 def searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         walk=True):
                     self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
                     self.__checkPath()
                     self.__findDataForDates()
                     self.__selectDataForTimes()
                     for i in range(len(self.filenameList)):
                         print("%s" %(self.filenameList[i]))
                     return
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print("No more Files")
                             return 0
                         filename = self.filenameList[idFile]
                         amisrFilePointer = h5py.File(filename,'r')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.amisrFilePointer = amisrFilePointer
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def __setNextFileOnline(self):
                     filename = self.filenameList[0]
                     if self.__filename_online != None:
                         self.__selectDataForTimes(online=True)
                         filename = self.filenameList[0]
                         while self.__filename_online == filename:
                             print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
                             sleep(self.__waitForNewFile)
                             self.__selectDataForTimes(online=True)
                             filename = self.filenameList[0]
                     self.__filename_online = filename
                     self.amisrFilePointer = h5py.File(filename,'r')
                     self.flagIsNewFile = 1
                     self.filename = filename
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def __readHeader(self):
                     self.radacHeaderObj = RadacHeader(self.amisrFilePointer)
                     #update values from experiment cfg file
                     if self.radacHeaderObj.nrecords == self.recordsperfile_fromfile:
                         self.radacHeaderObj.nrecords = self.recordsperfile_fromfile
                     self.radacHeaderObj.nbeams = self.nbeamcodes_fromfile
                     self.radacHeaderObj.npulses = self.npulsesint_fromfile
                     self.radacHeaderObj.nsamples = self.ngates_fromfile
                     #looking index list for data
                     start_index = self.radacHeaderObj.pulseCount[0,:][0]
                     end_index = self.radacHeaderObj.npulses
                     range4data = list(range(start_index, end_index))
                     self.index4_schain_datablock = numpy.array(range4data)
                     buffer_start_index = 0
                     buffer_end_index = self.radacHeaderObj.pulseCount[0,:][0]
                     range4buffer = list(range(buffer_start_index, buffer_end_index))
                     self.index4_buffer = numpy.array(range4buffer)
                     self.linear_pulseCount = numpy.array(range4data + range4buffer)
                     self.npulseByFrame = max(self.radacHeaderObj.pulseCount[0,:]+1)
                     #get tuning frequency
                     frequency_h5file_dataset = self.amisrFilePointer.get('Rx'+'/TuningFrequency')
                     self.frequency_h5file = frequency_h5file_dataset[0,0]
                     self.flagIsNewFile = 1
                 def __getBeamCode(self):
                     self.beamCodeDict = {}
                     self.beamRangeDict = {}
                     beamCodeMap = self.amisrFilePointer.get('Setup/BeamcodeMap')
                     for i in range(len(self.radacHeaderObj.beamCode[0,:])):
                         self.beamCodeDict.setdefault(i)
                         self.beamRangeDict.setdefault(i)
                         beamcodeValue = self.radacHeaderObj.beamCode[0,i]
                         beamcodeIndex = numpy.where(beamCodeMap[:,0] == beamcodeValue)[0][0]
                         x = beamCodeMap[beamcodeIndex][1]
                         y = beamCodeMap[beamcodeIndex][2]
                         z = beamCodeMap[beamcodeIndex][3]
                         self.beamCodeDict[i] = [beamcodeValue, x, y, z]
                     just4record0 = self.radacHeaderObj.beamCodeByPulse[0,:]
                     for i in range(len(list(self.beamCodeDict.values()))):
                         xx = numpy.where(just4record0==list(self.beamCodeDict.values())[i][0])
                         indexPulseByBeam = self.linear_pulseCount[xx[0]]
                         self.beamRangeDict[i] = indexPulseByBeam
                 def __getExpParameters(self):
                     if not(self.status):
                         return None
                     experimentCfgPath = os.path.join(self.path, self.dirnameList[0], 'Setup')
                     expFinder = glob.glob1(experimentCfgPath,'*.exp')
                     if len(expFinder)== 0:
                         self.status = 0
                         return None
                     experimentFilename = os.path.join(experimentCfgPath,expFinder[0])
                     f = open(experimentFilename)
                     lines = f.readlines()
                     f.close()
                     parmsList = ['npulsesint*','recordsperfile*','nbeamcodes*','ngates*']
                     filterList = [fnmatch.filter(lines, x) for x in parmsList]
                     values = [re.sub(r'\D',"",x[0]) for x in filterList]
                     self.npulsesint_fromfile = int(values[0])
                     self.recordsperfile_fromfile = int(values[1])
                     self.nbeamcodes_fromfile = int(values[2])
                     self.ngates_fromfile = int(values[3])
                     tufileFinder = fnmatch.filter(lines, 'tufile=*')
                     tufile = tufileFinder[0].split('=')[1].split('\n')[0]
                     tufile = tufile.split('\r')[0]
                     tufilename = os.path.join(experimentCfgPath,tufile)
                     f = open(tufilename)
                     lines = f.readlines()
                     f.close()
                     self.ippSeconds_fromfile = float(lines[1].split()[2])/1E6
                     self.status = 1
                 def __setIdsAndArrays(self):
                     self.dataByFrame = self.__setDataByFrame()
                     self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[0, :]
                     self.readRanges()
                     self.index_amisr_sample, self.index_amisr_buffer = self.radacHeaderObj.getIndexRangeToPulse(0)
                     self.radacTimeByFrame = numpy.zeros(self.radacHeaderObj.npulses)
                     if len(self.index_amisr_buffer) > 0:
                         self.buffer_radactime = numpy.zeros_like(self.radacTimeByFrame)
                 def __setNextFile(self,online=False):
                     if not(online):
                         newFile = self.__setNextFileOffline()
                     else:
                         newFile = self.__setNextFileOnline()
                     if not(newFile):
                         return 0
                     self.__readHeader()
                     if self.__firstFile:
                         self.__setIdsAndArrays()
                         self.__firstFile = False
                     self.__getBeamCode()
                     self.readDataBlock()
                 def setup(self,path=None,
                                 startDate=None,
                                 endDate=None,
                                 startTime=datetime.time(0,0,0),
                                 endTime=datetime.time(23,59,59),
                                 walk=True,
                                 timezone='ut',
                                 all=0,
                                 online=False):
                     self.timezone = timezone
                     self.all = all
                     self.online = online
                     if not(online):
                         #Busqueda de archivos offline
                         self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
                     else:
                         self.searchFilesOnLine(path, walk)
                     if not(self.filenameList):
                         print("There is no files into the folder: %s"%(path))
                         sys.exit(-1)
                     self.__getExpParameters()
                     self.fileIndex = -1
                     self.__setNextFile(online)
             #         first_beamcode = self.radacHeaderObj.beamCodeByPulse[0,0]
             #         index = numpy.where(self.radacHeaderObj.beamCodeByPulse[0,:]!=first_beamcode)[0][0]
                     self.profileIndex_offset = self.radacHeaderObj.pulseCount[0,:][0]
                     self.profileIndex = self.profileIndex_offset
                 def readRanges(self):
                     dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
                     self.rangeFromFile = numpy.reshape(dataset.value,(-1))
                     return self.rangeFromFile
                 def readRadacTime(self,idrecord, range1, range2):
                     self.radacTimeFromFile = self.radacHeaderObj.radacTime.value
                     radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
                     #radacTimeByFrame = dataset[idrecord - 1,range1]
                     #radacTimeByFrame = dataset[idrecord,range2]
                     return radacTimeByFrame
                 def readBeamCode(self, idrecord, range1, range2):
                     dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')
                     beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
                     self.beamCodesFromFile = dataset.value
                     #beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
                     #beamcodeByFrame[range2] = dataset[idrecord, range2]
                     beamcodeByFrame[range1] = dataset[idrecord, range1]
                     beamcodeByFrame[range2] = dataset[idrecord, range2]
                     return beamcodeByFrame
                 def __setDataByFrame(self):
                     ndata = 2 # porque es complejo
                     dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))
                     return dataByFrame
                 def __readDataSet(self):
                     dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
                     return dataset
                 def __setDataBlock(self,):
                     real = self.dataByFrame[:,:,0] #asumo que 0 es real
                     imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario
                     datablock = real + imag*1j #armo el complejo
                     return datablock
                 def readSamples_version1(self,idrecord):
                     #estas tres primeras lineas solo se deben ejecutar una vez
                     if self.flagIsNewFile:
                         #reading dataset
                         self.dataset = self.__readDataSet()
                         self.flagIsNewFile = 0
                     if idrecord == 0:
                         self.dataByFrame[self.index4_schain_datablock, : ,:] = self.dataset[0, self.index_amisr_sample,:,:]
                         self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[0, self.index_amisr_sample]
                         datablock = self.__setDataBlock()
                         if len(self.index_amisr_buffer) > 0:
                             self.buffer = self.dataset[0, self.index_amisr_buffer,:,:]
                             self.buffer_radactime = self.radacHeaderObj.radacTime[0, self.index_amisr_buffer]
                         return datablock
                     if len(self.index_amisr_buffer) > 0:
                         self.dataByFrame[self.index4_buffer,:,:] = self.buffer.copy()
                         self.radacTimeByFrame[self.index4_buffer] = self.buffer_radactime.copy()
                     self.dataByFrame[self.index4_schain_datablock,:,:] = self.dataset[idrecord, self.index_amisr_sample,:,:]
                     self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_sample]
                     datablock = self.__setDataBlock()
                     if len(self.index_amisr_buffer) > 0:
                         self.buffer = self.dataset[idrecord, self.index_amisr_buffer, :, :]
                         self.buffer_radactime = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_buffer]
                     return datablock
                 def readSamples(self,idrecord):
                     if self.flagIsNewFile:
                         self.dataByFrame = self.__setDataByFrame()
                         self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]
                         #reading ranges
                         self.readRanges()
                         #reading dataset
                         self.dataset = self.__readDataSet()
                     self.flagIsNewFile = 0
                     self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]
                     self.dataByFrame = self.dataset[idrecord, :, :, :]
                     datablock = self.__setDataBlock()
                     return datablock
                 def readDataBlock(self):
                     self.datablock = self.readSamples_version1(self.idrecord_count)
                     #self.datablock = self.readSamples(self.idrecord_count)
                     #print 'record:', self.idrecord_count
                     self.idrecord_count += 1
                     self.profileIndex = 0
                     if self.idrecord_count >= self.radacHeaderObj.nrecords:
                         self.idrecord_count = 0
                         self.flagIsNewFile = 1
                 def readNextBlock(self):
                     self.readDataBlock()
                     if self.flagIsNewFile:
                         self.__setNextFile(self.online)
                     pass
                 def __hasNotDataInBuffer(self):
                     #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
                     if self.profileIndex >= self.radacHeaderObj.npulses:
                         return 1
                     return 0
                 def printUTC(self):
                     print(self.dataOut.utctime)
                     print('')
                 def setObjProperties(self):
                     self.dataOut.heightList = self.rangeFromFile/1000.0 #km
                     self.dataOut.nProfiles = self.radacHeaderObj.npulses
                     self.dataOut.nRecords = self.radacHeaderObj.nrecords
                     self.dataOut.nBeams = self.radacHeaderObj.nbeams
                     self.dataOut.ippSeconds = self.ippSeconds_fromfile
             #         self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.frequency_h5file
                     self.dataOut.npulseByFrame = self.npulseByFrame
                     self.dataOut.nBaud = None
                     self.dataOut.nCode = None
                     self.dataOut.code = None
                     self.dataOut.beamCodeDict = self.beamCodeDict
                     self.dataOut.beamRangeDict = self.beamRangeDict
                     if self.timezone == 'lt':
                         self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
                     else:
                         self.dataOut.timeZone = 0 #by default time is UTC
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
-                        print('Process finished')
                         return 0
                     if self.__hasNotDataInBuffer():
                         self.readNextBlock()
                     if self.datablock is None: # setear esta condicion cuando no hayan datos por leers
                         self.dataOut.flagNoData = True
                         return 0
                     self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))
                     self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]
                     self.dataOut.profileIndex = self.profileIndex
                     self.dataOut.flagNoData = False
                     self.profileIndex += 1
                     return self.dataOut.data
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.setObjProperties()
                         self.isConfig = True
                     self.getData()

schainpy/model/io/jroIO_base.py +33 -33

             '''
             Created on Jul 2, 2014
             @author: roj-idl71
             '''
             import os
             import sys
             import glob
             import time
             import numpy
             import fnmatch
             import inspect
             import time
             import datetime
             import traceback
             import zmq
-            try:
-                from gevent import sleep
-            except:
-                from time import sleep
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
             from schainpy.utils import log
             import schainpy.admin
             LOCALTIME = True
             def isNumber(cad):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float(cad)
                     return True
                 except:
                     return False
             def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
                 """
                 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                     endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 try:
                     fp = open(filename, 'rb')
                 except IOError:
                     print("The file %s can't be opened" % (filename))
                     return 0
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 if not(sts):
                     print("Skipping the file %s because it has not a valid header" % (filename))
                     return 0
                 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                     return 0
                 return 1
             def isTimeInRange(thisTime, startTime, endTime):
                 if endTime >= startTime:
                     if (thisTime < startTime) or (thisTime > endTime):
                         return 0
                     return 1
                 else:
                     if (thisTime < startTime) and (thisTime > endTime):
                         return 0
                     return 1
             def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                     startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                     endTime            :    tiempo final del rango seleccionado en formato datetime.time
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 try:
                     fp = open(filename, 'rb')
                 except IOError:
                     print("The file %s can't be opened" % (filename))
                     return None
                 firstBasicHeaderObj = BasicHeader(LOCALTIME)
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
                 processingHeaderObj = ProcessingHeader()
                 lastBasicHeaderObj = BasicHeader(LOCALTIME)
                 sts = firstBasicHeaderObj.read(fp)
                 if not(sts):
                     print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
                     return None
                 if not systemHeaderObj.read(fp):
                     return None
                 if not radarControllerHeaderObj.read(fp):
                     return None
                 if not processingHeaderObj.read(fp):
                     return None
                 filesize = os.path.getsize(filename)
                 offset = processingHeaderObj.blockSize + 24  # header size
                 if filesize <= offset:
                     print("[Reading] %s: This file has not enough data" % filename)
                     return None
                 fp.seek(-offset, 2)
                 sts = lastBasicHeaderObj.read(fp)
                 fp.close()
                 thisDatetime = lastBasicHeaderObj.datatime
                 thisTime_last_block = thisDatetime.time()
                 thisDatetime = firstBasicHeaderObj.datatime
                 thisDate = thisDatetime.date()
                 thisTime_first_block = thisDatetime.time()
                 # General case
                 #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                 #-----------o----------------------------o-----------
                 #       startTime                     endTime
                 if endTime >= startTime:
                     if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
                         return None
                     return thisDatetime
                 # If endTime < startTime then endTime belongs to the next day
                 #<<<<<<<<<<<o                            o>>>>>>>>>>>
                 #-----------o----------------------------o-----------
                 #        endTime                    startTime
                 if (thisDate == startDate) and (thisTime_last_block < startTime):
                     return None
                 if (thisDate == endDate) and (thisTime_first_block > endTime):
                     return None
                 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
                     return None
                 return thisDatetime
             def isFolderInDateRange(folder, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     folder            :    nombre completo del directorio.
                                            Su formato deberia ser "/path_root/?YYYYDDD"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el directorio no tiene el formato adecuado
                 """
                 basename = os.path.basename(folder)
                 if not isRadarFolder(basename):
                     print("The folder %s has not the rigth format" % folder)
                     return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFolder(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def isFileInDateRange(filename, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                                            Su formato deberia ser "?YYYYDDDsss"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                                                 sss     :    set
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no tiene el formato adecuado
                 """
                 basename = os.path.basename(filename)
                 if not isRadarFile(basename):
                     print("The filename %s has not the rigth format" % filename)
                     return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFile(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def getFileFromSet(path, ext, set):
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     try:
                         year = int(thisFile[1:5])
                         doy = int(thisFile[5:8])
                     except:
                         continue
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 myfile = fnmatch.filter(
                     validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
                 if len(myfile) != 0:
                     return myfile[0]
                 else:
                     filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
                     print('the filename %s does not exist' % filename)
                     print('...going to the last file: ')
                 if validFilelist:
                     validFilelist = sorted(validFilelist, key=str.lower)
                     return validFilelist[-1]
                 return None
             def getlastFileFromPath(path, ext):
                 """
                 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     year = thisFile[1:5]
                     if not isNumber(year):
                         continue
                     doy = thisFile[5:8]
                     if not isNumber(doy):
                         continue
                     year = int(year)
                     doy = int(doy)
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 if validFilelist:
                     validFilelist = sorted(validFilelist, key=str.lower)
                     return validFilelist[-1]
                 return None
             def checkForRealPath(path, foldercounter, year, doy, set, ext):
                 """
                 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
                 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
                 el path exacto de un determinado file.
                 Example    :
                     nombre correcto del file es  .../.../D2009307/P2009307367.ext
                     Entonces la funcion prueba con las siguientes combinaciones
                         .../.../y2009307367.ext
                         .../.../Y2009307367.ext
                         .../.../x2009307/y2009307367.ext
                         .../.../x2009307/Y2009307367.ext
                         .../.../X2009307/y2009307367.ext
                         .../.../X2009307/Y2009307367.ext
                     siendo para este caso, la ultima combinacion de letras, identica al file buscado
                 Return:
                     Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
                     caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
                     para el filename
                 """
                 fullfilename = None
                 find_flag = False
                 filename = None
                 prefixDirList = [None, 'd', 'D']
                 if ext.lower() == ".r":  # voltage
                     prefixFileList = ['d', 'D']
                 elif ext.lower() == ".pdata":  # spectra
                     prefixFileList = ['p', 'P']
                 else:
                     return None, filename
                 # barrido por las combinaciones posibles
                 for prefixDir in prefixDirList:
                     thispath = path
                     if prefixDir != None:
                         # formo el nombre del directorio xYYYYDDD (x=d o x=D)
                         if foldercounter == 0:
                             thispath = os.path.join(path, "%s%04d%03d" %
                                                     (prefixDir, year, doy))
                         else:
                             thispath = os.path.join(path, "%s%04d%03d_%02d" % (
                                 prefixDir, year, doy, foldercounter))
                     for prefixFile in prefixFileList:  # barrido por las dos combinaciones posibles de "D"
                         # formo el nombre del file xYYYYDDDSSS.ext
                         filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
                         fullfilename = os.path.join(
                             thispath, filename)  # formo el path completo
                         if os.path.exists(fullfilename):  # verifico que exista
                             find_flag = True
                             break
                     if find_flag:
                         break
                 if not(find_flag):
                     return None, filename
                 return fullfilename, filename
             def isRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return 0
                 return 1
             def isRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
                 except:
                     return 0
                 return 1
             def getDateFromRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
                 return thisDate
             def getDateFromRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
                 return thisDate
             class JRODataIO:
                 c = 3E8
                 isConfig = False
                 basicHeaderObj = None
                 systemHeaderObj = None
                 radarControllerHeaderObj = None
                 processingHeaderObj = None
                 dtype = None
                 pathList = []
                 filenameList = []
                 filename = None
                 ext = None
                 flagIsNewFile = 1
                 flagDiscontinuousBlock = 0
                 flagIsNewBlock = 0
                 fp = None
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 versionFile = 1103
                 fileSize = None
             #     ippSeconds = None
                 fileSizeByHeader = None
                 fileIndex = None
                 profileIndex = None
                 blockIndex = None
                 nTotalBlocks = None
                 maxTimeStep = 30
                 lastUTTime = None
                 datablock = None
                 dataOut = None
                 blocksize = None
                 getByBlock = False
                 def __init__(self):
                     raise NotImplementedError
                 def run(self):
                     raise NotImplementedError
                 def getDtypeWidth(self):
                     dtype_index = get_dtype_index(self.dtype)
                     dtype_width = get_dtype_width(dtype_index)
                     return dtype_width
                 def getAllowedArgs(self):
                     if hasattr(self, '__attrs__'):
                         return self.__attrs__
                     else:
                         return inspect.getargspec(self.run).args
             class JRODataReader(JRODataIO):
                 online = 0
                 realtime = 0
                 nReadBlocks = 0
                 delay = 10  # number of seconds waiting a new file
                 nTries = 3  # quantity tries
                 nFiles = 3  # number of files for searching
                 path = None
                 foldercounter = 0
                 flagNoMoreFiles = 0
                 datetimeList = []
                 __isFirstTimeOnline = 1
                 __printInfo = True
                 profileIndex = None
                 nTxs = 1
                 txIndex = None
                 # Added--------------------
                 selBlocksize = None
                 selBlocktime = None
                 def __init__(self):
                     """
                     This class is used to find data files
                     Example:
                         reader = JRODataReader()
                         fileList = reader.findDataFiles()
                     """
                     pass
                 def createObjByDefault(self):
                     """
                     """
                     raise NotImplementedError
                 def getBlockDimension(self):
                     raise NotImplementedError
                 def searchFilesOffLine(self,
                                        path,
                                        startDate=None,
                                        endDate=None,
                                        startTime=datetime.time(0, 0, 0),
                                        endTime=datetime.time(23, 59, 59),
                                        set=None,
                                        expLabel='',
                                        ext='.r',
                                        cursor=None,
                                        skip=None,
                                        walk=True):
                     self.filenameList = []
                     self.datetimeList = []
                     pathList = []
                     dateList, pathList = self.findDatafiles(
                         path, startDate, endDate, expLabel, ext, walk, include_path=True)
                     if dateList == []:
                         return [], []
                     if len(dateList) > 1:
                         print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
                     else:
                         print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
                     filenameList = []
                     datetimeList = []
                     for thisPath in pathList:
                         fileList = glob.glob1(thisPath, "*%s" % ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath, file)
                             if not isFileInDateRange(filename, startDate, endDate):
                                 continue
                             thisDatetime = isFileInTimeRange(
                                 filename, startDate, endDate, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if cursor is not None and skip is not None:
                         filenameList = filenameList[cursor * skip:cursor * skip + skip]
                         datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
                     if not(filenameList):
                         print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
                         return [], []
                     print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
                     # for i in range(len(filenameList)):
                     #     print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
                     """
                     Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
                     devuelve el archivo encontrado ademas de otros datos.
                     Input:
                         path           :    carpeta donde estan contenidos los files que contiene data
                         expLabel        :     Nombre del subexperimento (subfolder)
                         ext              :    extension de los files
                         walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                     Return:
                         directory   :    eL directorio donde esta el file encontrado
                         filename    :    el ultimo file de una determinada carpeta
                         year        :    el anho
                         doy         :    el numero de dia del anho
                         set         :    el set del archivo
                     """
                     if not os.path.isdir(path):
                         return None, None, None, None, None, None
                     dirList = []
                     if not walk:
                         fullpath = path
                         foldercounter = 0
                     else:
                         # Filtra solo los directorios
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path, thisPath)):
                                 continue
                             if not isRadarFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None, None, None, None, None
                         dirList = sorted(dirList, key=str.lower)
                         doypath = dirList[-1]
                         foldercounter = int(doypath.split('_')[1]) if len(
                             doypath.split('_')) > 1 else 0
                         fullpath = os.path.join(path, doypath, expLabel)
                     print("[Reading] %s folder was found: " % (fullpath))
                     if set == None:
                         filename = getlastFileFromPath(fullpath, ext)
                     else:
                         filename = getFileFromSet(fullpath, ext, set)
                     if not(filename):
                         return None, None, None, None, None, None
                     print("[Reading] %s file was found" % (filename))
                     if not(self.__verifyFile(os.path.join(fullpath, filename))):
                         return None, None, None, None, None, None
                     year = int(filename[1:5])
                     doy = int(filename[5:8])
                     set = int(filename[8:11])
                     return fullpath, foldercounter, filename, year, doy, set
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
-            #                 print "[Reading] No more Files"
                             return 0
                         filename = self.filenameList[idFile]
                         if not(self.__verifyFile(filename)):
                             continue
                         fileSize = os.path.getsize(filename)
                         fp = open(filename, 'rb')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fp = fp
             #         print "[Reading] Setting the file: %s"%self.filename
                     return 1
                 def __setNextFileOnline(self):
                     """
                     Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
                     no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
                     siguientes.
                     Affected:
                         self.flagIsNewFile
                         self.filename
                         self.fileSize
                         self.fp
                         self.set
                         self.flagNoMoreFiles
                     Return:
 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
 : si el file fue abierto con exito y esta listo a ser leido
                     Excepciones:
                         Si un determinado file no puede ser abierto
                     """
                     nFiles = 0
                     fileOk_flag = False
                     firstTime_flag = True
                     self.set += 1
                     if self.set > 999:
                         self.set = 0
                         self.foldercounter += 1
                     # busca el 1er file disponible
                     fullfilename, filename = checkForRealPath(
                         self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
                     if fullfilename:
                         if self.__verifyFile(fullfilename, False):
                             fileOk_flag = True
                     # si no encuentra un file entonces espera y vuelve a buscar
                     if not(fileOk_flag):
                         # busco en los siguientes self.nFiles+1 files posibles
                         for nFiles in range(self.nFiles + 1):
                             if firstTime_flag:  # si es la 1era vez entonces hace el for self.nTries veces
                                 tries = self.nTries
                             else:
                                 tries = 1  # si no es la 1era vez entonces solo lo hace una vez
                             for nTries in range(tries):
                                 if firstTime_flag:
-                                    print("\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1))
+                                    log.warning(
-                                    sleep(self.delay)
+                                        "Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1),
+                                        self.name)
+                                    time.sleep(self.delay)
                                 else:
-                                    print("\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext))
+                                    log.warning(
+                                        "Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext),
+                                        self.name)
                                 fullfilename, filename = checkForRealPath(
                                     self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
                                 if fullfilename:
                                     if self.__verifyFile(fullfilename):
                                         fileOk_flag = True
                                         break
                             if fileOk_flag:
                                 break
                             firstTime_flag = False
-                            log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
+                            log.warning(
+                                'Skipping the file {} due to this file doesn\'t exist'.format(filename),
+                                self.name)
                             self.set += 1
                             # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
                             if nFiles == (self.nFiles - 1):
                                 self.set = 0
                                 self.doy += 1
                                 self.foldercounter = 0
                     if fileOk_flag:
                         self.fileSize = os.path.getsize(fullfilename)
                         self.filename = fullfilename
                         self.flagIsNewFile = 1
                         if self.fp != None:
                             self.fp.close()
                         self.fp = open(fullfilename, 'rb')
                         self.flagNoMoreFiles = 0
-            #             print '[Reading] Setting the file: %s' % fullfilename
                     else:
+                        raise schainpy.admin.SchainError('Time for waiting new files reach')
                         self.fileSize = 0
                         self.filename = None
                         self.flagIsNewFile = 0
                         self.fp = None
                         self.flagNoMoreFiles = 1
-            #             print '[Reading] No more files to read'
                     return fileOk_flag
                 def setNextFile(self):
                     if self.fp != None:
                         self.fp.close()
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
-                        self.dataOut.error = 'No more files to read'
+                        raise schainpy.admin.SchainWarning('No more files to read')
-                        return 0
                     if self.verbose:
                         print('[Reading] Setting the file: %s' % self.filename)
                     self.__readFirstHeader()
                     self.nReadBlocks = 0
                     return 1
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range(self.nTries):
                         self.fp.close()
                         self.fp = open(self.filename, 'rb')
                         self.fp.seek(currentPointer)
                         self.fileSize = os.path.getsize(self.filename)
                         currentSize = self.fileSize - currentPointer
                         if (currentSize >= neededSize):
                             self.basicHeaderObj.read(self.fp)
                             return 1
                         if self.fileSize == self.fileSizeByHeader:
                             #                self.flagEoF = True
                             return 0
                         print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
-                        sleep(self.delay)
+                        time.sleep(self.delay)
                     return 0
                 def waitDataBlock(self, pointer_location, blocksize=None):
                     currentPointer = pointer_location
                     if blocksize is None:
                         neededSize = self.processingHeaderObj.blockSize  # + self.basicHeaderSize
                     else:
                         neededSize = blocksize
                     for nTries in range(self.nTries):
                         self.fp.close()
                         self.fp = open(self.filename, 'rb')
                         self.fp.seek(currentPointer)
                         self.fileSize = os.path.getsize(self.filename)
                         currentSize = self.fileSize - currentPointer
                         if (currentSize >= neededSize):
                             return 1
                         log.warning(
                             "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
                             self.name
                             )
-                        sleep(self.delay)
+                        time.sleep(self.delay)
                     return 0
                 def __jumpToLastBlock(self):
                     if not(self.__isFirstTimeOnline):
                         return
                     csize = self.fileSize - self.fp.tell()
                     blocksize = self.processingHeaderObj.blockSize
                     # salta el primer bloque de datos
                     if csize > self.processingHeaderObj.blockSize:
                         self.fp.seek(self.fp.tell() + blocksize)
                     else:
                         return
                     csize = self.fileSize - self.fp.tell()
                     neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     while True:
                         if self.fp.tell() < self.fileSize:
                             self.fp.seek(self.fp.tell() + neededsize)
                         else:
                             self.fp.seek(self.fp.tell() - neededsize)
                             break
             #        csize = self.fileSize - self.fp.tell()
             #        neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
             #        factor = int(csize/neededsize)
             #        if factor > 0:
             #            self.fp.seek(self.fp.tell() + factor*neededsize)
                     self.flagIsNewFile = 0
                     self.__isFirstTimeOnline = 0
                 def __setNewBlock(self):
                     # if self.server is None:
                     if self.fp == None:
                         return 0
             #         if self.online:
             #             self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if self.realtime:
                         self.flagDiscontinuousBlock = 1
                         if not(self.setNextFile()):
                             return 0
                         else:
                             return 1
                     # if self.server is None:
                     currentSize = self.fileSize - self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     if (currentSize >= neededSize):
                         self.basicHeaderObj.read(self.fp)
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     # else:
                     #     self.basicHeaderObj.read(self.zHeader)
                     #     self.lastUTTime = self.basicHeaderObj.utc
                     #     return 1
                     if self.__waitNewBlock():
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     # if self.server is None:
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime
                     self.lastUTTime = self.basicHeaderObj.utc
                     self.flagDiscontinuousBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagDiscontinuousBlock = 1
                     return 1
                 def readNextBlock(self):
                     # Skip block out of startTime and endTime
                     while True:
                         if not(self.__setNewBlock()):
-                            self.dataOut.error = 'No more files to read'
+                            raise schainpy.admin.SchainWarning('No more files to read')
-                            return 0
                         if not(self.readBlock()):
                             return 0
                         self.getBasicHeader()
                         if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
                             print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
                                                                                   self.processingHeaderObj.dataBlocksPerFile,
                                                                                   self.dataOut.datatime.ctime()))
                             continue
                         break
                     if self.verbose:
                         print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
                                                                    self.processingHeaderObj.dataBlocksPerFile,
                                                                    self.dataOut.datatime.ctime()))
                     return 1
                 def __readFirstHeader(self):
                     self.basicHeaderObj.read(self.fp)
                     self.systemHeaderObj.read(self.fp)
                     self.radarControllerHeaderObj.read(self.fp)
                     self.processingHeaderObj.read(self.fp)
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
                                                PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
                     else:
                         raise ValueError('Data type was not defined')
                     self.dtype = datatype_str
                     #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
                         self.firstHeaderSize + self.basicHeaderSize * \
                         (self.processingHeaderObj.dataBlocksPerFile - 1)
                     #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
                     #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def __verifyFile(self, filename, msgFlag=True):
                     msg = None
                     try:
                         fp = open(filename, 'rb')
                     except IOError:
                         if msgFlag:
                             print("[Reading] File %s can't be opened" % (filename))
                         return False
                     currentPosition = fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
                     if neededSize == 0:
                         basicHeaderObj = BasicHeader(LOCALTIME)
                         systemHeaderObj = SystemHeader()
                         radarControllerHeaderObj = RadarControllerHeader()
                         processingHeaderObj = ProcessingHeader()
                         if not(basicHeaderObj.read(fp)):
                             fp.close()
                             return False
                         if not(systemHeaderObj.read(fp)):
                             fp.close()
                             return False
                         if not(radarControllerHeaderObj.read(fp)):
                             fp.close()
                             return False
                         if not(processingHeaderObj.read(fp)):
                             fp.close()
                             return False
                         neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
                     else:
                         msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
                     fp.close()
                     fileSize = os.path.getsize(filename)
                     currentSize = fileSize - currentPosition
                     if currentSize < neededSize:
                         if msgFlag and (msg != None):
                             print(msg)
                         return False
                     return True
                 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
                     path_empty = True
                     dateList = []
                     pathList = []
                     multi_path = path.split(',')
                     if not walk:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             fileList = glob.glob1(single_path, "*" + ext)
                             if not fileList:
                                 continue
                             path_empty = False
                             fileList.sort()
                             for thisFile in fileList:
                                 if not os.path.isfile(os.path.join(single_path, thisFile)):
                                     continue
                                 if not isRadarFile(thisFile):
                                     continue
                                 if not isFileInDateRange(thisFile, startDate, endDate):
                                     continue
                                 thisDate = getDateFromRadarFile(thisFile)
                                 if thisDate in dateList or single_path in pathList:
                                     continue
                                 dateList.append(thisDate)
                                 pathList.append(single_path)
                     else:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             dirList = []
                             for thisPath in os.listdir(single_path):
                                 if not os.path.isdir(os.path.join(single_path, thisPath)):
                                     continue
                                 if not isRadarFolder(thisPath):
                                     continue
                                 if not isFolderInDateRange(thisPath, startDate, endDate):
                                     continue
                                 dirList.append(thisPath)
                             if not dirList:
                                 continue
                             dirList.sort()
                             for thisDir in dirList:
                                 datapath = os.path.join(single_path, thisDir, expLabel)
                                 fileList = glob.glob1(datapath, "*" + ext)
                                 if not fileList:
                                     continue
                                 path_empty = False
                                 thisDate = getDateFromRadarFolder(thisDir)
                                 pathList.append(datapath)
                                 dateList.append(thisDate)
                     dateList.sort()
                     if walk:
                         pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
                     else:
                         pattern_path = multi_path[0]
                     if path_empty:
-                        print("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
+                        raise schainpy.admin.SchainError("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
                     else:
                         if not dateList:
-                            print("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
+                            raise schainpy.admin.SchainError("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
                     if include_path:
                         return dateList, pathList
                     return dateList
                 def setup(self,
                           path=None,
                           startDate=None,
                           endDate=None,
                           startTime=datetime.time(0, 0, 0),
                           endTime=datetime.time(23, 59, 59),
                           set=None,
                           expLabel="",
                           ext=None,
                           online=False,
                           delay=60,
                           walk=True,
                           getblock=False,
                           nTxs=1,
                           realtime=False,
                           blocksize=None,
                           blocktime=None,
                           skip=None,
                           cursor=None,
                           warnings=True,
                           verbose=True,
                           server=None,
                           format=None,
                           oneDDict=None,
                           twoDDict=None,
                           independentParam=None):
+                    self.online = online
+                    self.realtime = realtime
+                    self.delay = delay
+                    self.getByBlock = getblock
+                    self.nTxs = nTxs
+                    self.startTime = startTime
+                    self.endTime = endTime
+                    self.endDate = endDate
+                    self.startDate = startDate
                     if server is not None:
                         if 'tcp://' in server:
                             address = server
                         else:
                             address = 'ipc:///tmp/%s' % server
                         self.server = address
                         self.context = zmq.Context()
                         self.receiver = self.context.socket(zmq.PULL)
                         self.receiver.connect(self.server)
                         time.sleep(0.5)
                         print('[Starting] ReceiverData from {}'.format(self.server))
                     else:
                         self.server = None
                         if path == None:
                             raise ValueError("[Reading] The path is not valid")
                         if ext == None:
                             ext = self.ext
                         if online:
                             print("[Reading] Searching files in online mode...")
                             for nTries in range(self.nTries):
                                 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
                                     path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
                                 if fullpath:
                                     break
                                 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
-                                sleep(self.delay)
+                                time.sleep(self.delay)
                             if not(fullpath):
-                                self.dataOut.error = 'There isn\'t any valid file in {}'.format(path)
+                                raise schainpy.admin.SchainError('There isn\'t any valid file in {}'.format(path))
                                 return
                             self.year = year
                             self.doy = doy
                             self.set = set - 1
                             self.path = path
                             self.foldercounter = foldercounter
                             last_set = None
                         else:
                             print("[Reading] Searching files in offline mode ...")
                             pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                              startTime=startTime, endTime=endTime,
                                                                              set=set, expLabel=expLabel, ext=ext,
                                                                              walk=walk, cursor=cursor,
                                                                              skip=skip)
                             if not(pathList):
                                 self.fileIndex = -1
                                 self.pathList = []
                                 self.filenameList = []
                                 return
                             self.fileIndex = -1
                             self.pathList = pathList
                             self.filenameList = filenameList
                             file_name = os.path.basename(filenameList[-1])
                             basename, ext = os.path.splitext(file_name)
                             last_set = int(basename[-3:])
-                        self.online = online
-                        self.realtime = realtime
-                        self.delay = delay
                         ext = ext.lower()
                         self.ext = ext
-                        self.getByBlock = getblock
-                        self.nTxs = nTxs
-                        self.startTime = startTime
-                        self.endTime = endTime
-                        self.endDate = endDate
-                        self.startDate = startDate
                         # Added-----------------
                         self.selBlocksize = blocksize
                         self.selBlocktime = blocktime
                         # Verbose-----------
                         self.verbose = verbose
                         self.warnings = warnings
                         if not(self.setNextFile()):
                             if (startDate != None) and (endDate != None):
                                 print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
                             elif startDate != None:
                                 print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
                             else:
                                 print("[Reading] No files")
                                 self.fileIndex = -1
                                 self.pathList = []
                                 self.filenameList = []
                                 return
-                #         self.getBasicHeader()
                         if last_set != None:
                             self.dataOut.last_block = last_set * \
                                 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
                     return
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
 . + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                     self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
             #         self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
                 def getFirstHeader(self):
                     raise NotImplementedError
                 def getData(self):
                     raise NotImplementedError
                 def hasNotDataInBuffer(self):
                     raise NotImplementedError
                 def readBlock(self):
                     raise NotImplementedError
                 def isEndProcess(self):
                     return self.flagNoMoreFiles
                 def printReadBlocks(self):
                     print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
                 def printTotalBlocks(self):
                     print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
                 def printNumberOfBlock(self):
                     'SPAM!'
             #         if self.flagIsNewBlock:
             #             print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
             #                                                       self.processingHeaderObj.dataBlocksPerFile,
             #                                                       self.dataOut.datatime.ctime())
                 def printInfo(self):
                     if self.__printInfo == False:
                         return
                     self.basicHeaderObj.printInfo()
                     self.systemHeaderObj.printInfo()
                     self.radarControllerHeaderObj.printInfo()
                     self.processingHeaderObj.printInfo()
                     self.__printInfo = False
                 def run(self,
                         path=None,
                         startDate=None,
                         endDate=None,
                         startTime=datetime.time(0, 0, 0),
                         endTime=datetime.time(23, 59, 59),
                         set=None,
                         expLabel="",
                         ext=None,
                         online=False,
                         delay=60,
                         walk=True,
                         getblock=False,
                         nTxs=1,
                         realtime=False,
                         blocksize=None,
                         blocktime=None,
                         skip=None,
                         cursor=None,
                         warnings=True,
                         server=None,
                         verbose=True,
                         format=None,
                         oneDDict=None,
                         twoDDict=None,
                         independentParam=None, **kwargs):
                     if not(self.isConfig):
                         self.setup(path=path,
                                    startDate=startDate,
                                    endDate=endDate,
                                    startTime=startTime,
                                    endTime=endTime,
                                    set=set,
                                    expLabel=expLabel,
                                    ext=ext,
                                    online=online,
                                    delay=delay,
                                    walk=walk,
                                    getblock=getblock,
                                    nTxs=nTxs,
                                    realtime=realtime,
                                    blocksize=blocksize,
                                    blocktime=blocktime,
                                    skip=skip,
                                    cursor=cursor,
                                    warnings=warnings,
                                    server=server,
                                    verbose=verbose,
                                    format=format,
                                    oneDDict=oneDDict,
                                    twoDDict=twoDDict,
                                    independentParam=independentParam)
                         self.isConfig = True
                     if server is None:
                         self.getData()
                     else:
                         self.getFromServer()
             class JRODataWriter(JRODataIO):
                 """
                 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 blockIndex = 0
                 path = None
                 setFile = None
                 profilesPerBlock = None
                 blocksPerFile = None
                 nWriteBlocks = 0
                 fileDate = None
                 def __init__(self, dataOut=None):
                     raise NotImplementedError
                 def hasAllDataInBuffer(self):
                     raise NotImplementedError
                 def setBlockDimension(self):
                     raise NotImplementedError
                 def writeBlock(self):
                     raise NotImplementedError
                 def putData(self):
                     raise NotImplementedError
                 def getProcessFlags(self):
                     processFlags = 0
                     dtype_index = get_dtype_index(self.dtype)
                     procflag_dtype = get_procflag_dtype(dtype_index)
                     processFlags += procflag_dtype
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code is not None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nCohInt > 1:
                         processFlags += PROCFLAG.COHERENT_INTEGRATION
                     if self.dataOut.type == "Spectra":
                         if self.dataOut.nIncohInt > 1:
                             processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                         if self.dataOut.data_dc is not None:
                             processFlags += PROCFLAG.SAVE_CHANNELS_DC
                         if self.dataOut.flagShiftFFT:
                             processFlags += PROCFLAG.SHIFT_FFT_DATA
                     return processFlags
                 def setBasicHeader(self):
                     self.basicHeaderObj.size = self.basicHeaderSize  # bytes
                     self.basicHeaderObj.version = self.versionFile
                     self.basicHeaderObj.dataBlock = self.nTotalBlocks
                     utc = numpy.floor(self.dataOut.utctime)
                     milisecond = (self.dataOut.utctime - utc) * 1000.0
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = milisecond
                     self.basicHeaderObj.timeZone = self.dataOut.timeZone
                     self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                     self.basicHeaderObj.errorCount = self.dataOut.errorCount
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj self.
                     Return:
                         None
                     """
                     raise NotImplementedError
                 def __writeFirstHeader(self):
                     """
                     Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                     Affected:
                         __dataType
                     Return:
                         None
                     """
             #        CALCULAR PARAMETROS
                     sizeLongHeader = self.systemHeaderObj.size + \
                         self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                     self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                     self.basicHeaderObj.write(self.fp)
                     self.systemHeaderObj.write(self.fp)
                     self.radarControllerHeaderObj.write(self.fp)
                     self.processingHeaderObj.write(self.fp)
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                     """
                     if self.fp == None:
                         self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                         self.basicHeaderObj.write(self.fp)
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     """
                     Selecciona el bloque siguiente de datos y los escribe en un file
                     Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                     """
                     if not(self.__setNewBlock()):
                         return 0
                     self.writeBlock()
                     print("[Writing] Block No. %d/%d" % (self.blockIndex,
                                                          self.processingHeaderObj.dataBlocksPerFile))
                     return 1
                 def setNextFile(self):
                     """
                     Determina el siguiente file que sera escrito
                     Affected:
                         self.filename
                         self.subfolder
                         self.fp
                         self.setFile
                         self.flagIsNewFile
                     Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                     """
                     ext = self.ext
                     path = self.path
                     if self.fp != None:
                         self.fp.close()
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
                     fullpath = os.path.join(path, subfolder)
                     setFile = self.setFile
                     if not(os.path.exists(fullpath)):
                         os.mkdir(fullpath)
                         setFile = -1  # inicializo mi contador de seteo
                     else:
                         filesList = os.listdir(fullpath)
                         if len(filesList) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 # inicializo mi contador de seteo al seteo del ultimo file
                                 setFile = int(filen[8:11])
                             else:
                                 setFile = -1
                         else:
                             setFile = -1  # inicializo mi contador de seteo
                     setFile += 1
                     # If this is a new day it resets some values
                     if self.dataOut.datatime.date() > self.fileDate:
                         setFile = 0
                         self.nTotalBlocks = 0
                     filen = '{}{:04d}{:03d}{:03d}{}'.format(
                             self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
                     filename = os.path.join(path, subfolder, filen)
                     fp = open(filename, 'wb')
                     self.blockIndex = 0
                     # guardando atributos
                     self.filename = filename
                     self.subfolder = subfolder
                     self.fp = fp
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     self.fileDate = self.dataOut.datatime.date()
                     self.setFirstHeader()
                     print('[Writing] Opening file: %s' % self.filename)
                     self.__writeFirstHeader()
                     return 1
                 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
                     """
                     Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                     Inputs:
                         path                :    directory where data will be saved
                         profilesPerBlock    :    number of profiles per block
                         set                 :    initial file set
                         datatype            :    An integer number that defines data type:
 : int8  (1 byte)
 : int16 (2 bytes)
 : int32 (4 bytes)
 : int64 (8 bytes)
 : float32 (4 bytes)
 : double64 (8 bytes)
                     Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                     """
                     if ext == None:
                         ext = self.ext
                     self.ext = ext.lower()
                     self.path = path
                     if set is None:
                         self.setFile = -1
                     else:
                         self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile
                     self.profilesPerBlock = profilesPerBlock
                     self.dataOut = dataOut
                     self.fileDate = self.dataOut.datatime.date()
                     # By default
                     self.dtype = self.dataOut.dtype
                     if datatype is not None:
                         self.dtype = get_numpy_dtype(datatype)
                     if not(self.setNextFile()):
                         print("[Writing] There isn't a next file")
                         return 0
                     self.setBlockDimension()
                     return 1
                 def run(self, dataOut, path, blocksPerFile=100, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
                                    set=set, ext=ext, datatype=datatype, **kwargs)
                         self.isConfig = True
                     self.dataOut = dataOut
                     self.putData()
                     return self.dataOut

schainpy/model/io/jroIO_digitalRF.py +9 -13

             '''
             Created on Jul 3, 2014
             @author: roj-idl71
             '''
             # SUBCHANNELS EN VEZ DE CHANNELS
             # BENCHMARKS -> PROBLEMAS CON ARCHIVOS GRANDES -> INCONSTANTE EN EL TIEMPO
             # ACTUALIZACION DE VERSION
             # HEADERS
             # MODULO DE ESCRITURA
             # METADATA
             import os
+            import time
             import datetime
             import numpy
             import timeit
             from fractions import Fraction
-            try:
+            import schainpy.admin
-                from gevent import sleep
-            except:
-                from time import sleep
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
-            from time import time
             import pickle
             try:
                 import digital_rf
             except:
                 print('You should install "digital_rf" module if you want to read Digital RF data')
             @MPDecorator
             class DigitalRFReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self)
                     self.dataOut = Voltage()
                     self.__printInfo = True
                     self.__flagDiscontinuousBlock = False
                     self.__bufferIndex = 9999999
                     self.__ippKm = None
                     self.__codeType = 0
                     self.__nCode = None
                     self.__nBaud = None
                     self.__code = None
                     self.dtype = None
                     self.oldAverage = None
                 def close(self):
                     print('Average of writing to digital rf format is ', self.oldAverage * 1000)
                     return
                 def __getCurrentSecond(self):
                     return self.__thisUnixSample / self.__sample_rate
                 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
                 def __setFileHeader(self):
                     '''
                     In this method will be initialized every parameter of dataOut object (header, no data)
                     '''
                     ippSeconds = 1.0 * self.__nSamples / self.__sample_rate
                     nProfiles = 1.0 / ippSeconds  # Number of profiles in one second
                     try:
                         self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
                             self.__radarControllerHeader)
                     except:
                         self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
                             txA=0,
                             txB=0,
                             nWindows=1,
                             nHeights=self.__nSamples,
                             firstHeight=self.__firstHeigth,
                             deltaHeight=self.__deltaHeigth,
                             codeType=self.__codeType,
                             nCode=self.__nCode, nBaud=self.__nBaud,
                             code=self.__code)
                     try:
                         self.dataOut.systemHeaderObj = SystemHeader(self.__systemHeader)
                     except:
                         self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                     nProfiles=nProfiles,
                                                                     nChannels=len(
                                                                         self.__channelList),
                                                                     adcResolution=14)
                     self.dataOut.type = "Voltage"
                     self.dataOut.data = None
                     self.dataOut.dtype = self.dtype
                     # self.dataOut.nChannels = 0
                     # self.dataOut.nHeights = 0
                     self.dataOut.nProfiles = int(nProfiles)
                     self.dataOut.heightList = self.__firstHeigth + \
                         numpy.arange(self.__nSamples, dtype=numpy.float) * \
                         self.__deltaHeigth
                     self.dataOut.channelList = list(range(self.__num_subchannels))
                     self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
                     # self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData = True
                     self.dataOut.flagDataAsBlock = False
                     # Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime = None
                     # timezone like jroheader, difference in minutes between UTC and localtime
                     self.dataOut.timeZone = self.__timezone / 60
                     self.dataOut.dstFlag = 0
                     self.dataOut.errorCount = 0
                     try:
                         self.dataOut.nCohInt = self.fixed_metadata_dict.get(
                             'nCohInt', self.nCohInt)
                         # asumo que la data esta decodificada
                         self.dataOut.flagDecodeData = self.fixed_metadata_dict.get(
                             'flagDecodeData', self.flagDecodeData)
                         # asumo que la data esta sin flip
                         self.dataOut.flagDeflipData = self.fixed_metadata_dict['flagDeflipData']
                         self.dataOut.flagShiftFFT = self.fixed_metadata_dict['flagShiftFFT']
                         self.dataOut.useLocalTime = self.fixed_metadata_dict['useLocalTime']
                     except:
                         pass
                     self.dataOut.ippSeconds = ippSeconds
                     # Time interval between profiles
                     # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.__frequency
                     self.dataOut.realtime = self.__online
                 def findDatafiles(self, path, startDate=None, endDate=None):
                     if not os.path.isdir(path):
                         return []
                     try:
                         digitalReadObj = digital_rf.DigitalRFReader(
                             path, load_all_metadata=True)
                     except:
                         digitalReadObj = digital_rf.DigitalRFReader(path)
                     channelNameList = digitalReadObj.get_channels()
                     if not channelNameList:
                         return []
                     metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
                     sample_rate = metadata_dict['sample_rate'][0]
                     this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
                     try:
                         timezone = this_metadata_file['timezone'].value
                     except:
                         timezone = 0
                     startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(
                         channelNameList[0]) / sample_rate - timezone
                     startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
                     endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
                     if not startDate:
                         startDate = startDatetime.date()
                     if not endDate:
                         endDate = endDatatime.date()
                     dateList = []
                     thisDatetime = startDatetime
                     while(thisDatetime <= endDatatime):
                         thisDate = thisDatetime.date()
                         if thisDate < startDate:
                             continue
                         if thisDate > endDate:
                             break
                         dateList.append(thisDate)
                         thisDatetime += datetime.timedelta(1)
                     return dateList
                 def setup(self, path=None,
                           startDate=None,
                           endDate=None,
                           startTime=datetime.time(0, 0, 0),
                           endTime=datetime.time(23, 59, 59),
                           channelList=None,
                           nSamples=None,
                           online=False,
                           delay=60,
                           buffer_size=1024,
                           ippKm=None,
                           nCohInt=1,
                           nCode=1,
                           nBaud=1,
                           flagDecodeData=False,
                           code=numpy.ones((1, 1), dtype=numpy.int),
                           **kwargs):
                     '''
                     In this method we should set all initial parameters.
                     Inputs:
                         path
                         startDate
                         endDate
                         startTime
                         endTime
                         set
                         expLabel
                         ext
                         online
                         delay
                     '''
                     self.nCohInt = nCohInt
                     self.flagDecodeData = flagDecodeData
                     self.i = 0
                     if not os.path.isdir(path):
                         raise ValueError("[Reading] Directory %s does not exist" % path)
                     try:
                         self.digitalReadObj = digital_rf.DigitalRFReader(
                             path, load_all_metadata=True)
                     except:
                         self.digitalReadObj = digital_rf.DigitalRFReader(path)
                     channelNameList = self.digitalReadObj.get_channels()
                     if not channelNameList:
                         raise ValueError("[Reading] Directory %s does not have any files" % path)
                     if not channelList:
                         channelList = list(range(len(channelNameList)))
                     ##########  Reading metadata ######################
                     top_properties = self.digitalReadObj.get_properties(
                         channelNameList[channelList[0]])
                     self.__num_subchannels = top_properties['num_subchannels']
                     self.__sample_rate = 1.0 * \
                         top_properties['sample_rate_numerator'] / \
                         top_properties['sample_rate_denominator']
                     # self.__samples_per_file = top_properties['samples_per_file'][0]
                     self.__deltaHeigth = 1e6 * 0.15 / self.__sample_rate  # why 0.15?
                     this_metadata_file = self.digitalReadObj.get_digital_metadata(
                         channelNameList[channelList[0]])
                     metadata_bounds = this_metadata_file.get_bounds()
                     self.fixed_metadata_dict = this_metadata_file.read(
                         metadata_bounds[0])[metadata_bounds[0]]  # GET FIRST HEADER
                     try:
                         self.__processingHeader = self.fixed_metadata_dict['processingHeader']
                         self.__radarControllerHeader = self.fixed_metadata_dict['radarControllerHeader']
                         self.__systemHeader = self.fixed_metadata_dict['systemHeader']
                         self.dtype = pickle.loads(self.fixed_metadata_dict['dtype'])
                     except:
                         pass
                     self.__frequency = None
                     self.__frequency = self.fixed_metadata_dict.get('frequency', 1)
                     self.__timezone = self.fixed_metadata_dict.get('timezone', 300)
                     try:
                         nSamples = self.fixed_metadata_dict['nSamples']
                     except:
                         nSamples = None
                     self.__firstHeigth = 0
                     try:
                         codeType = self.__radarControllerHeader['codeType']
                     except:
                         codeType = 0
                     try:
                         if codeType:
                             nCode = self.__radarControllerHeader['nCode']
                             nBaud = self.__radarControllerHeader['nBaud']
                             code = self.__radarControllerHeader['code']
                     except:
                         pass
                     if not ippKm:
                         try:
                             # seconds to km
                             ippKm = self.__radarControllerHeader['ipp']
                         except:
                             ippKm = None
                     ####################################################
                     self.__ippKm = ippKm
                     startUTCSecond = None
                     endUTCSecond = None
                     if startDate:
                         startDatetime = datetime.datetime.combine(startDate, startTime)
                         startUTCSecond = (
                             startDatetime - datetime.datetime(1970, 1, 1)).total_seconds() + self.__timezone
                     if endDate:
                         endDatetime = datetime.datetime.combine(endDate, endTime)
                         endUTCSecond = (endDatetime - datetime.datetime(1970,
 , 1)).total_seconds() + self.__timezone
                     start_index, end_index = self.digitalReadObj.get_bounds(
                         channelNameList[channelList[0]])
                     if not startUTCSecond:
                         startUTCSecond = start_index / self.__sample_rate
                     if start_index > startUTCSecond * self.__sample_rate:
                         startUTCSecond = start_index / self.__sample_rate
                     if not endUTCSecond:
                         endUTCSecond = end_index / self.__sample_rate
                     if end_index < endUTCSecond * self.__sample_rate:
                         endUTCSecond = end_index / self.__sample_rate
                     if not nSamples:
                         if not ippKm:
                             raise ValueError("[Reading] nSamples or ippKm should be defined")
                         nSamples = int(ippKm / (1e6 * 0.15 / self.__sample_rate))
                     channelBoundList = []
                     channelNameListFiltered = []
                     for thisIndexChannel in channelList:
                         thisChannelName = channelNameList[thisIndexChannel]
                         start_index, end_index = self.digitalReadObj.get_bounds(
                             thisChannelName)
                         channelBoundList.append((start_index, end_index))
                         channelNameListFiltered.append(thisChannelName)
                     self.profileIndex = 0
                     self.i = 0
                     self.__delay = delay
                     self.__codeType = codeType
                     self.__nCode = nCode
                     self.__nBaud = nBaud
                     self.__code = code
                     self.__datapath = path
                     self.__online = online
                     self.__channelList = channelList
                     self.__channelNameList = channelNameListFiltered
                     self.__channelBoundList = channelBoundList
                     self.__nSamples = nSamples
                     self.__samples_to_read = int(nSamples)  # FIJO: AHORA 40
                     self.__nChannels = len(self.__channelList)
                     self.__startUTCSecond = startUTCSecond
                     self.__endUTCSecond = endUTCSecond
                     self.__timeInterval = 1.0 * self.__samples_to_read / \
                         self.__sample_rate  # Time interval
                     if online:
                         # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
                         startUTCSecond = numpy.floor(endUTCSecond)
                     # por que en el otro metodo lo primero q se hace es sumar samplestoread
                     self.__thisUnixSample = int(
                         startUTCSecond * self.__sample_rate) - self.__samples_to_read
                     self.__data_buffer = numpy.zeros(
                         (self.__num_subchannels, self.__samples_to_read), dtype=numpy.complex)
                     self.__setFileHeader()
                     self.isConfig = True
                     print("[Reading] Digital RF Data was found from %s to %s " % (
                         datetime.datetime.utcfromtimestamp(
                             self.__startUTCSecond - self.__timezone),
                         datetime.datetime.utcfromtimestamp(
                             self.__endUTCSecond - self.__timezone)
                     ))
                     print("[Reading] Starting process from %s to %s" % (datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
                                                                         datetime.datetime.utcfromtimestamp(
                         endUTCSecond - self.__timezone)
                     ))
                     self.oldAverage = None
                     self.count = 0
                     self.executionTime = 0
                 def __reload(self):
                     #         print
                     #         print "%s not in range [%s, %s]" %(
                     #                                           datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                     #                                           datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
                     #                                           datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
                     #                                           )
                     print("[Reading] reloading metadata ...")
                     try:
                         self.digitalReadObj.reload(complete_update=True)
                     except:
                         self.digitalReadObj.reload()
                     start_index, end_index = self.digitalReadObj.get_bounds(
                         self.__channelNameList[self.__channelList[0]])
                     if start_index > self.__startUTCSecond * self.__sample_rate:
                         self.__startUTCSecond = 1.0 * start_index / self.__sample_rate
                     if end_index > self.__endUTCSecond * self.__sample_rate:
                         self.__endUTCSecond = 1.0 * end_index / self.__sample_rate
                         print()
                         print("[Reading] New timerange found [%s, %s] " % (
                             datetime.datetime.utcfromtimestamp(
                                 self.__startUTCSecond - self.__timezone),
                             datetime.datetime.utcfromtimestamp(
                                 self.__endUTCSecond - self.__timezone)
                         ))
                         return True
                     return False
                 def timeit(self, toExecute):
-                    t0 = time()
+                    t0 = time.time()
                     toExecute()
-                    self.executionTime = time() - t0
+                    self.executionTime = time.time() - t0
                     if self.oldAverage is None:
                         self.oldAverage = self.executionTime
                     self.oldAverage = (self.executionTime + self.count *
                                        self.oldAverage) / (self.count + 1.0)
                     self.count = self.count + 1.0
                     return
                 def __readNextBlock(self, seconds=30, volt_scale=1):
                     '''
                     '''
                     # Set the next data
                     self.__flagDiscontinuousBlock = False
                     self.__thisUnixSample += self.__samples_to_read
                     if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
                         print("[Reading] There are no more data into selected time-range")
                         if self.__online:
                             self.__reload()
                         else:
                             return False
                         if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
                             return False
                             self.__thisUnixSample -= self.__samples_to_read
                     indexChannel = 0
                     dataOk = False
                     for thisChannelName in self.__channelNameList:  # TODO VARIOS CHANNELS?
                         for indexSubchannel in range(self.__num_subchannels):
                             try:
                                 t0 = time()
                                 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
                                                                               self.__samples_to_read,
                                                                               thisChannelName, sub_channel=indexSubchannel)
                                 self.executionTime = time() - t0
                                 if self.oldAverage is None:
                                     self.oldAverage = self.executionTime
                                 self.oldAverage = (
                                     self.executionTime + self.count * self.oldAverage) / (self.count + 1.0)
                                 self.count = self.count + 1.0
                             except IOError as e:
                                 # read next profile
                                 self.__flagDiscontinuousBlock = True
                                 print("[Reading] %s" % datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e)
                                 break
                             if result.shape[0] != self.__samples_to_read:
                                 self.__flagDiscontinuousBlock = True
                                 print("[Reading] %s: Too few samples were found, just %d/%d  samples" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                                                          result.shape[0],
                                                                                                          self.__samples_to_read))
                                 break
                             self.__data_buffer[indexSubchannel, :] = result * volt_scale
                             indexChannel += 1
                             dataOk = True
                     self.__utctime = self.__thisUnixSample / self.__sample_rate
                     if not dataOk:
                         return False
                     print("[Reading] %s: %d samples <> %f sec" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                   self.__samples_to_read,
                                                                   self.__timeInterval))
                     self.__bufferIndex = 0
                     return True
                 def __isBufferEmpty(self):
                     return self.__bufferIndex > self.__samples_to_read - self.__nSamples  # 40960 - 40
                 def getData(self, seconds=30, nTries=5):
                     '''
                         This method gets the data from files and put the data into the dataOut object
                         In addition, increase el the buffer counter in one.
                         Return:
                             data    :    retorna un perfil de voltages (alturas * canales) copiados desde el
                                         buffer. Si no hay mas archivos a leer retorna None.
                         Affected:
                             self.dataOut
                             self.profileIndex
                             self.flagDiscontinuousBlock
                             self.flagIsNewBlock
                     '''
                     err_counter = 0
                     self.dataOut.flagNoData = True
                     if self.__isBufferEmpty():
                         self.__flagDiscontinuousBlock = False
                         while True:
                             if self.__readNextBlock():
                                 break
                             if self.__thisUnixSample > self.__endUTCSecond * self.__sample_rate:
-                                self.dataOut.error = 'Error'
+                                raise schainpy.admin.SchainError('Error')
                                 return
                             if self.__flagDiscontinuousBlock:
-                                self.dataOut.error = 'discontinuous block found'
+                                raise schainpy.admin.SchainError('discontinuous block found')
                                 return
                             if not self.__online:
-                                self.dataOut.error = 'Online?'
+                                raise schainpy.admin.SchainError('Online?')
                                 return
                             err_counter += 1
                             if err_counter > nTries:
-                                self.dataOut.error = 'Max retrys reach'
+                                raise schainpy.admin.SchainError('Max retrys reach')
                                 return
                             print('[Reading] waiting %d seconds to read a new block' % seconds)
-                            sleep(seconds)
+                            time.sleep(seconds)
                     self.dataOut.data = self.__data_buffer[:,
                                                            self.__bufferIndex:self.__bufferIndex + self.__nSamples]
                     self.dataOut.utctime = (
                         self.__thisUnixSample + self.__bufferIndex) / self.__sample_rate
                     self.dataOut.flagNoData = False
                     self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
                     self.dataOut.profileIndex = self.profileIndex
                     self.__bufferIndex += self.__nSamples
                     self.profileIndex += 1
                     if self.profileIndex == self.dataOut.nProfiles:
                         self.profileIndex = 0
                     return
                 def printInfo(self):
                     '''
                     '''
                     if self.__printInfo == False:
                         return
                     # self.systemHeaderObj.printInfo()
                     # self.radarControllerHeaderObj.printInfo()
                     self.__printInfo = False
                 def printNumberOfBlock(self):
                     '''
                     '''
                     return
                     # print self.profileIndex
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                     #self.i = self.i+1
                     self.getData(seconds=self.__delay)
                     return
             class DigitalRFWriter(Operation):
                 '''
                 classdocs
                 '''
                 def __init__(self, **kwargs):
                     '''
                     Constructor
                     '''
                     Operation.__init__(self, **kwargs)
                     self.metadata_dict = {}
                     self.dataOut = None
                     self.dtype = None
                     self.oldAverage = 0
                 def setHeader(self):
                     self.metadata_dict['frequency'] = self.dataOut.frequency
                     self.metadata_dict['timezone'] = self.dataOut.timeZone
                     self.metadata_dict['dtype'] = pickle.dumps(self.dataOut.dtype)
                     self.metadata_dict['nProfiles'] = self.dataOut.nProfiles
                     self.metadata_dict['heightList'] = self.dataOut.heightList
                     self.metadata_dict['channelList'] = self.dataOut.channelList
                     self.metadata_dict['flagDecodeData'] = self.dataOut.flagDecodeData
                     self.metadata_dict['flagDeflipData'] = self.dataOut.flagDeflipData
                     self.metadata_dict['flagShiftFFT'] = self.dataOut.flagShiftFFT
                     self.metadata_dict['useLocalTime'] = self.dataOut.useLocalTime
                     self.metadata_dict['nCohInt'] = self.dataOut.nCohInt
                     self.metadata_dict['type'] = self.dataOut.type
                     self.metadata_dict['flagDataAsBlock'] = getattr(
                         self.dataOut, 'flagDataAsBlock', None)  # chequear
                 def setup(self, dataOut, path, frequency, fileCadence, dirCadence, metadataCadence, set=0, metadataFile='metadata', ext='.h5'):
                     '''
                     In this method we should set all initial parameters.
                     Input:
                         dataOut: Input data will also be outputa data
                     '''
                     self.setHeader()
                     self.__ippSeconds = dataOut.ippSeconds
                     self.__deltaH = dataOut.getDeltaH()
                     self.__sample_rate = 1e6 * 0.15 / self.__deltaH
                     self.__dtype = dataOut.dtype
                     if len(dataOut.dtype) == 2:
                         self.__dtype = dataOut.dtype[0]
                     self.__nSamples = dataOut.systemHeaderObj.nSamples
                     self.__nProfiles = dataOut.nProfiles
                     if self.dataOut.type != 'Voltage':
                         raise 'Digital RF cannot be used with this data type'
                         self.arr_data = numpy.ones((1, dataOut.nFFTPoints * len(
                             self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
                     else:
                         self.arr_data = numpy.ones((self.__nSamples, len(
                             self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
                     file_cadence_millisecs = 1000
                     sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator()
                     sample_rate_numerator = int(sample_rate_fraction.numerator)
                     sample_rate_denominator = int(sample_rate_fraction.denominator)
                     start_global_index = dataOut.utctime * self.__sample_rate
                     uuid = 'prueba'
                     compression_level = 0
                     checksum = False
                     is_complex = True
                     num_subchannels = len(dataOut.channelList)
                     is_continuous = True
                     marching_periods = False
                     self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, dirCadence,
                                                                       fileCadence, start_global_index,
                                                                       sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum,
                                                                       is_complex, num_subchannels, is_continuous, marching_periods)
                     metadata_dir = os.path.join(path, 'metadata')
                     os.system('mkdir %s' % (metadata_dir))
                     self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, dirCadence, 1,  # 236, file_cadence_millisecs / 1000
                                                                                     sample_rate_numerator, sample_rate_denominator,
                                                                                     metadataFile)
                     self.isConfig = True
                     self.currentSample = 0
                     self.oldAverage = 0
                     self.count = 0
                     return
                 def writeMetadata(self):
                     start_idx = self.__sample_rate * self.dataOut.utctime
                     self.metadata_dict['processingHeader'] = self.dataOut.processingHeaderObj.getAsDict(
                     )
                     self.metadata_dict['radarControllerHeader'] = self.dataOut.radarControllerHeaderObj.getAsDict(
                     )
                     self.metadata_dict['systemHeader'] = self.dataOut.systemHeaderObj.getAsDict(
                     )
                     self.digitalMetadataWriteObj.write(start_idx, self.metadata_dict)
                     return
                 def timeit(self, toExecute):
                     t0 = time()
                     toExecute()
                     self.executionTime = time() - t0
                     if self.oldAverage is None:
                         self.oldAverage = self.executionTime
                     self.oldAverage = (self.executionTime + self.count *
                                        self.oldAverage) / (self.count + 1.0)
                     self.count = self.count + 1.0
                     return
                 def writeData(self):
                     if self.dataOut.type != 'Voltage':
                         raise 'Digital RF cannot be used with this data type'
                         for channel in self.dataOut.channelList:
                             for i in range(self.dataOut.nFFTPoints):
                                 self.arr_data[1][channel * self.dataOut.nFFTPoints +
                                                  i]['r'] = self.dataOut.data[channel][i].real
                                 self.arr_data[1][channel * self.dataOut.nFFTPoints +
                                                  i]['i'] = self.dataOut.data[channel][i].imag
                     else:
                         for i in range(self.dataOut.systemHeaderObj.nSamples):
                             for channel in self.dataOut.channelList:
                                 self.arr_data[i][channel]['r'] = self.dataOut.data[channel][i].real
                                 self.arr_data[i][channel]['i'] = self.dataOut.data[channel][i].imag
                     def f(): return self.digitalWriteObj.rf_write(self.arr_data)
                     self.timeit(f)
                     return
                 def run(self, dataOut, frequency=49.92e6, path=None, fileCadence=1000, dirCadence=36000, metadataCadence=1, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     Inputs:
                         dataOut: object with the data
                     '''
                     # print dataOut.__dict__
                     self.dataOut = dataOut
                     if not self.isConfig:
                         self.setup(dataOut, path, frequency, fileCadence,
                                    dirCadence, metadataCadence, **kwargs)
                         self.writeMetadata()
                     self.writeData()
                     ## self.currentSample += 1
                     # if self.dataOut.flagDataAsBlock or self.currentSample == 1:
                     # self.writeMetadata()
                     ## if self.currentSample == self.__nProfiles: self.currentSample = 0
                     return dataOut
                 def close(self):
                     print('[Writing] - Closing files ')
                     print('Average of writing to digital rf format is ', self.oldAverage * 1000)
                     try:
                         self.digitalWriteObj.close()
                     except:
                         pass

schainpy/model/io/jroIO_hf.py 0 -1

             '''
             Created on Jul 3, 2014
             @author: roj-com0419
             '''
             import os,sys
             import time,datetime
             import h5py
             import numpy
             import fnmatch
             import re
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             def isNumber(str):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float( str )
                     return True
                 except:
                     return False
             def getFileFromSet(path, ext, set=None):
                 validFilelist = []
                 fileList = os.listdir(path)
                 if len(fileList) < 1:
                     return None
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     try:
                         number= int(thisFile[6:16])
                  #       year = int(thisFile[1:5])
                  #       doy  = int(thisFile[5:8])
                     except:
                         continue
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 if len(validFilelist) < 1:
                     return None
                 validFilelist = sorted( validFilelist, key=str.lower )
                 if set == None:
                     return validFilelist[-1]
                 print("set =" ,set)
                 for thisFile in validFilelist:
                     if set <= int(thisFile[6:16]):
                         print(thisFile,int(thisFile[6:16]))
                         return thisFile
                 return validFilelist[-1]
                 myfile = fnmatch.filter(validFilelist,'*%10d*'%(set))
                 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
                 if len(myfile)!= 0:
                     return myfile[0]
                 else:
                     filename = '*%10.10d%s'%(set,ext.lower())
                     print('the filename %s does not exist'%filename)
                     print('...going to the last file: ')
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             def getlastFileFromPath(path, ext):
                 """
             Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     try:
                         number= int(thisFile[6:16])
                     except:
                         print("There is a file or folder with different format")
                     if not isNumber(number):
                         continue
             #        year = thisFile[1:5]
             #        if not isNumber(year):
             #            continue
             #       doy = thisFile[5:8]
             #        if not isNumber(doy):
             #            continue
                     number= int(number)
             #        year = int(year)
             #        doy = int(doy)
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 if validFilelist:
                     validFilelist = sorted( validFilelist, key=str.lower )
                     return validFilelist[-1]
                 return None
             class HFReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 path     = None
                 startDate= None
                 endDate  = None
                 startTime= None
                 endTime  = None
                 walk     = None
                 isConfig = False
                 dataOut=None
                 nTries = 3
                 ext      = ".hdf5"
                 def __init__(self, **kwargs):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self, **kwargs)
                     self.isConfig =False
                     self.datablock = None
                     self.filename_current=None
                     self.utc = 0
                     self.ext='.hdf5'
                     self.flagIsNewFile = 1
                     #-------------------------------------------------
                     self.fileIndex=None
                     self.profileIndex_offset=None
                     self.filenameList=[]
                     self.hfFilePointer= None
                     self.filename_online = None
                     self.status=True
                     self.flagNoMoreFiles= False
                     self.__waitForNewFile = 20
                     #--------------------------------------------------
                     self.dataOut = self.createObjByDefault()
                 def createObjByDefault(self):
                     dataObj = Voltage()
                     return dataObj
                 def setObjProperties(self):
                     pass
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.blocksize
                     Return:
                         None
                     """
                     pts2read =self.nChannels*self.nHeights*self.nProfiles
                     self.blocksize = pts2read
                 def __readHeader(self):
                     self.nProfiles = 100
                     self.nHeights = 1000
                     self.nChannels = 2
                     self.__firstHeigth=0
                     self.__nSamples=1000
                     self.__deltaHeigth=1.5
                     self.__sample_rate=1e5
                     #self.__frequency=2.72e6
                     #self.__frequency=3.64e6
                     self.__frequency=None
                     self.__online = False
                     self.filename_next_set=None
                     #print "Frequency of Operation:", self.__frequency
                 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.walk = walk
                 def __checkPath(self):
                     if os.path.exists(self.path):
                         self.status=1
                     else:
                         self.status=0
                         print('Path %s does not exits'%self.path)
                         return
                     return
                 def __selDates(self, hf_dirname_format):
                     try:
                         dir_hf_filename= self.path+"/"+hf_dirname_format
                         fp= h5py.File(dir_hf_filename,'r')
                         hipoc=fp['t'].value
                         fp.close()
                         date_time=datetime.datetime.utcfromtimestamp(hipoc)
                         year =int(date_time[0:4])
                         month=int(date_time[5:7])
                         dom  =int(date_time[8:10])
                         thisDate= datetime.date(year,month,dom)
                         if (thisDate>=self.startDate and thisDate <= self.endDate):
                             return hf_dirname_format
                     except:
                         return None
                 def __findDataForDates(self,online=False):
                     if not(self.status):
                         return None
                     pat = '\d+.\d+'
                     dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
                     dirnameList = [x for x in dirnameList if x!=None]
                     dirnameList = [x.string for x in dirnameList]
                     if not(online):
                         dirnameList = [self.__selDates(x) for x in dirnameList]
                         dirnameList = [x for x in dirnameList if x!=None]
                     if len(dirnameList)>0:
                             self.status = 1
                             self.dirnameList = dirnameList
                             self.dirnameList.sort()
                     else:
                         self.status = 0
                         return None
                 def __getTimeFromData(self):
                     startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
                     endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                     print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
                     print('........................................')
                     filter_filenameList=[]
                     self.filenameList.sort()
                     for i in range(len(self.filenameList)-1):
                         filename=self.filenameList[i]
                         dir_hf_filename= filename
                         fp= h5py.File(dir_hf_filename,'r')
                         hipoc=fp['t'].value
                         hipoc=hipoc+self.timezone
                         date_time=datetime.datetime.utcfromtimestamp(hipoc)
                         fp.close()
                         year =int(date_time[0:4])
                         month=int(date_time[5:7])
                         dom  =int(date_time[8:10])
                         hour =int(date_time[11:13])
                         min  =int(date_time[14:16])
                         sec  =int(date_time[17:19])
                         this_time=datetime.datetime(year,month,dom,hour,min,sec)
                         if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
                             filter_filenameList.append(filename)
                     filter_filenameList.sort()
                     self.filenameList = filter_filenameList
                     return 1
                 def __getFilenameList(self):
                     #print "hola"
                     #print self.dirnameList
                     dirList = [os.path.join(self.path,x) for x in self.dirnameList]
                     self.filenameList= dirList
                     #print self.filenameList
                     #print "pase",len(self.filenameList)
                 def __selectDataForTimes(self, online=False):
                     if not(self.status):
                         return None
                     #----------------
                     self.__getFilenameList()
                     #----------------
                     if not(online):
                         if not(self.all):
                             self.__getTimeFromData()
                         if len(self.filenameList)>0:
                             self.status=1
                             self.filenameList.sort()
                         else:
                             self.status=0
                             return None
                     else:
                         if self.set != None:
                             filename=getFileFromSet(self.path,self.ext,self.set)
                             if self.flag_nextfile==True:
                                 self.dirnameList=[filename]
                                 fullfilename=self.path+"/"+filename
                                 self.filenameList=[fullfilename]
                                 self.filename_next_set=int(filename[6:16])+10
                                 self.flag_nextfile=False
                             else:
                                 print(filename)
                                 print("PRIMERA CONDICION")
                                 #if self.filename_next_set== int(filename[6:16]):
                                 print("TODO BIEN")
                                 if filename == None:
                                     raise ValueError("corregir")
                                 self.dirnameList=[filename]
                                 fullfilename=self.path+"/"+filename
                                 self.filenameList=[fullfilename]
                                 self.filename_next_set=int(filename[6:16])+10
                                 print("Setting next file",self.filename_next_set)
                                 self.set=int(filename[6:16])
                                 if True:
                                     pass
                                 else:
                                     print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO")
                         else:
                             filename =getlastFileFromPath(self.path,self.ext)
                             if self.flag_nextfile==True:
                                 self.dirnameList=[filename]
                                 fullfilename=self.path+"/"+filename
                                 self.filenameList=[self.filenameList[-1]]
                                 self.filename_next_set=int(filename[6:16])+10
                                 self.flag_nextfile=False
                             else:
                                filename=getFileFromSet(self.path,self.ext,self.set)
                                print(filename)
                                print("PRIMERA CONDICION")
                                 #if self.filename_next_set== int(filename[6:16]):
                                print("TODO BIEN")
                                if filename == None:
                                    raise ValueError("corregir")
                                self.dirnameList=[filename]
                                fullfilename=self.path+"/"+filename
                                self.filenameList=[fullfilename]
                                self.filename_next_set=int(filename[6:16])+10
                                print("Setting next file",self.filename_next_set)
                                self.set=int(filename[6:16])
                                if True:
                                    pass
                                else:
                                    print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO")
                 def searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         ext,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         walk=True):
                     self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
                     self.__checkPath()
                     self.__findDataForDates()
                     #print self.dirnameList
                     self.__selectDataForTimes()
                     for i in range(len(self.filenameList)):
                         print("%s"% (self.filenameList[i]))
                     return
                 def searchFilesOnLine(self,
                                         path,
                                         expLabel= "",
                                         ext=None,
                                         startDate=None,
                                         endDate=None,
                                         walk=True,
                                         set=None):
                     startDate = datetime.datetime.utcnow().date()
                     endDate = datetime.datetime.utcnow().date()
                     self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
                     self.__checkPath()
                     fullpath=path
                     print("%s folder was found: " %(fullpath ))
                     if set == None:
                         self.set=None
                         filename =getlastFileFromPath(fullpath,ext)
                         startDate= datetime.datetime.utcnow().date
                         endDate= datetime.datetime.utcnow().date()
             #
                     else:
                         filename= getFileFromSet(fullpath,ext,set)
                         startDate=None
                         endDate=None
             #
                     if not (filename):
                         return None,None,None,None,None
                     #print "%s file was found" %(filename)
             #
             #         dir_hf_filename= self.path+"/"+filename
             #         fp= h5py.File(dir_hf_filename,'r')
             #         hipoc=fp['t'].value
             #         fp.close()
             #         date_time=datetime.datetime.utcfromtimestamp(hipoc)
             #
             #         year =int(date_time[0:4])
             #         month=int(date_time[5:7])
             #         dom  =int(date_time[8:10])
             #         set= int(filename[4:10])
             #         self.set=set-1
                     #self.dirnameList=[filename]
                     filenameList= fullpath+"/"+filename
                     self.dirnameList=[filename]
                     self.filenameList=[filenameList]
                     self.flag_nextfile=True
                     #self.__findDataForDates(online=True)
                     #self.dirnameList=[self.dirnameList[-1]]
                     #print self.dirnameList
                     #self.__selectDataForTimes(online=True)
                     #return fullpath,filename,year,month,dom,set
                     return
                 def __setNextFile(self,online=False):
                     """
                     """
                     if not(online):
                         newFile = self.__setNextFileOffline()
                     else:
                         newFile = self.__setNextFileOnline()
                     if not(newFile):
                         return 0
                     return 1
                 def __setNextFileOffline(self):
                     """
                     """
                     idFile= self.fileIndex
                     while(True):
                         idFile += 1
                         if not (idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print("No more Files")
                             return 0
                         filename = self.filenameList[idFile]
                         hfFilePointer =h5py.File(filename,'r')
                         epoc=hfFilePointer['t'].value
                         #this_time=datetime.datetime(year,month,dom,hour,min,sec)
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.hfFilePointer = hfFilePointer
                     hfFilePointer.close()
                     self.__t0=epoc
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def __setNextFileOnline(self):
                     """
                     """
                     print("SOY NONE",self.set)
                     if self.set==None:
                         pass
                     else:
                         self.set +=10
                     filename = self.filenameList[0]#fullfilename
                     if self.filename_online != None:
                         self.__selectDataForTimes(online=True)
                         filename = self.filenameList[0]
                         while self.filename_online == filename:
                             print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
                             time.sleep(self.__waitForNewFile)
                             #self.__findDataForDates(online=True)
                             self.set=self.filename_next_set
                             self.__selectDataForTimes(online=True)
                             filename = self.filenameList[0]
                             sizeoffile=os.path.getsize(filename)
                     #print filename
                     sizeoffile=os.path.getsize(filename)
                     if sizeoffile<1670240:
                         print("%s is not the rigth  size"%filename)
                         delay=50
                         print('waiting %d seconds for delay...'%(delay))
                         time.sleep(delay)
                     sizeoffile=os.path.getsize(filename)
                     if sizeoffile<1670240:
                         delay=50
                         print('waiting %d  more seconds for delay...'%(delay))
                         time.sleep(delay)
                     sizeoffile=os.path.getsize(filename)
                     if sizeoffile<1670240:
                         delay=50
                         print('waiting %d  more seconds for delay...'%(delay))
                         time.sleep(delay)
                     try:
                         hfFilePointer=h5py.File(filename,'r')
                     except:
                         print("Error reading file %s"%filename)
                     self.filename_online=filename
                     epoc=hfFilePointer['t'].value
                     self.hfFilePointer=hfFilePointer
                     hfFilePointer.close()
                     self.__t0=epoc
                     self.flagIsNewFile = 1
                     self.filename = filename
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def __getExpParameters(self):
                     if not(self.status):
                         return None
                 def setup(self,
                            path = None,
                            startDate = None,
                            endDate = None,
                            startTime = datetime.time(0,0,0),
                            endTime = datetime.time(23,59,59),
                            set = None,
                            expLabel = "",
                            ext = None,
                            all=0,
                            timezone=0,
                            online = False,
                            delay = 60,
                            walk = True):
                     '''
                     In this method we should set all initial parameters.
                     '''
                     if path==None:
                         raise ValueError("The path is not valid")
                     if ext==None:
                         ext = self.ext
                     self.timezone= timezone
                     self.online= online
                     self.all=all
                     #if set==None:
                     #print set
                     if not(online):
                         print("Searching files in offline mode...")
                         self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk)
                     else:
                         print("Searching files in online mode...")
                         self.searchFilesOnLine(path, walk,ext,set=set)
                         if set==None:
                             pass
                         else:
                             self.set=set-10
             #             for nTries in range(self.nTries):
             #
             #                 fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
             #
             #                 if fullpath:
             #                     break
             #                 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
             #                 time.sleep(self.delay)
             #             if not(fullpath):
             #                 print "There ins't valid files in %s" % path
             #                 return None
                     if not(self.filenameList):
                         print("There  is no files into the folder: %s"%(path))
                         sys.exit(-1)
                     self.__getExpParameters()
                     self.fileIndex = -1
                     self.__setNextFile(online)
                     self.__readMetadata()
                     self.__setLocalVariables()
                     self.__setHeaderDO()
                     #self.profileIndex_offset= 0
                     #self.profileIndex = self.profileIndex_offset
                     self.isConfig = True
                 def __readMetadata(self):
                     self.__readHeader()
                 def __setLocalVariables(self):
                     self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
                     #
                     self.profileIndex = 9999
                 def __setHeaderDO(self):
                     self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
                     self.dataOut.systemHeaderObj = SystemHeader()
                     #---------------------------------------------------------
                     self.dataOut.systemHeaderObj.nProfiles=100
                     self.dataOut.systemHeaderObj.nSamples=1000
                     SAMPLING_STRUCTURE=[('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')]
                     self.dataOut.radarControllerHeaderObj.samplingWindow=numpy.zeros((1,),SAMPLING_STRUCTURE)
                     self.dataOut.radarControllerHeaderObj.samplingWindow['h0']=0
                     self.dataOut.radarControllerHeaderObj.samplingWindow['dh']=1.5
                     self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']=1000
                     self.dataOut.radarControllerHeaderObj.nHeights=int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'])
                     self.dataOut.radarControllerHeaderObj.firstHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['h0']
                     self.dataOut.radarControllerHeaderObj.deltaHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['dh']
                     self.dataOut.radarControllerHeaderObj.samplesWin = self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']
                     self.dataOut.radarControllerHeaderObj.nWindows=1
                     self.dataOut.radarControllerHeaderObj.codetype=0
                     self.dataOut.radarControllerHeaderObj.numTaus=0
                     #self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4')
                     #self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4')
                     #self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4')
                     #self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0)
                     self.dataOut.radarControllerHeaderObj.code_size=0
                     self.dataOut.nBaud=0
                     self.dataOut.nCode=0
                     self.dataOut.nPairs=0
                     #---------------------------------------------------------
                     self.dataOut.type = "Voltage"
                     self.dataOut.data = None
                     self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
                     self.dataOut.nProfiles = 1
                     self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
                     self.dataOut.channelList = list(range(self.nChannels))
                     #self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData = True
                     #Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime = None
                     self.dataOut.timeZone = self.timezone
                     self.dataOut.dstFlag = 0
                     self.dataOut.errorCount = 0
                     self.dataOut.nCohInt = 1
                     self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
                     self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
                     self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
                     #Time interval between profiles
                     #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.__frequency
                     self.dataOut.realtime = self.__online
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= self.nProfiles:
                         return 1
                     return 0
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def __setNewBlock(self):
                     if self.hfFilePointer==None:
                         return 0
                     if self.flagIsNewFile:
                         return 1
                     if self.profileIndex < self.nProfiles:
                         return 1
                     self.__setNextFile(self.online)
                     return 1
                 def readBlock(self):
                     fp=h5py.File(self.filename,'r')
                                   #Puntero que apunta al archivo hdf5
                     ch0=(fp['ch0']).value    #Primer canal (100,1000)--(perfiles,alturas)
                     ch1=(fp['ch1']).value    #Segundo canal (100,1000)--(perfiles,alturas)
                     fp.close()
                     ch0= ch0.swapaxes(0,1)   #Primer canal (100,1000)--(alturas,perfiles)
                     ch1= ch1.swapaxes(0,1)   #Segundo canal (100,1000)--(alturas,perfiles)
                     self.datablock = numpy.array([ch0,ch1])
                     self.flagIsNewFile=0
                     self.profileIndex=0
                     return 1
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
-                        print('Process finished')
                         return 0
                     if self.__hasNotDataInBuffer():
                         if not(self.readNextBlock()):
                             self.dataOut.flagNodata=True
                             return 0
                     ##############################
                     ##############################
                     self.dataOut.data = self.datablock[:,:,self.profileIndex]
                     self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds*self.profileIndex
                     self.dataOut.profileIndex= self.profileIndex
                     self.dataOut.flagNoData=False
                     self.profileIndex +=1
                     return self.dataOut.data
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()

schainpy/model/io/jroIO_kamisr.py 0 -1

             '''
             Created on Set 9, 2015
             @author: roj-idl71 Karim Kuyeng
             '''
             import os
             import sys
             import glob
             import fnmatch
             import datetime
             import time
             import re
             import h5py
             import numpy
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             from numpy import imag
             class AMISRReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self)
                     self.set = None
                     self.subset = None
                     self.extension_file = '.h5'
                     self.dtc_str = 'dtc'
                     self.dtc_id = 0
                     self.status = True
                     self.isConfig = False
                     self.dirnameList = []
                     self.filenameList = []
                     self.fileIndex = None
                     self.flagNoMoreFiles = False
                     self.flagIsNewFile = 0
                     self.filename = ''
                     self.amisrFilePointer = None
                     self.dataset = None
                     self.profileIndex = 0
                     self.beamCodeByFrame = None
                     self.radacTimeByFrame = None
                     self.dataset = None
                     self.__firstFile = True
                     self.buffer = None
                     self.timezone = 'ut'
                     self.__waitForNewFile = 20
                     self.__filename_online = None
                     #Is really necessary create the output object in the initializer
                     self.dataOut = Voltage()
                 def setup(self,path=None,
                                 startDate=None,
                                 endDate=None,
                                 startTime=None,
                                 endTime=None,
                                 walk=True,
                                 timezone='ut',
                                 all=0,
                                 code = None,
                                 nCode = 0,
                                 nBaud = 0,
                                 online=False):
                     self.timezone = timezone
                     self.all = all
                     self.online = online
                     self.code = code
                     self.nCode = int(nCode)
                     self.nBaud = int(nBaud)
                     #self.findFiles()
                     if not(online):
                         #Busqueda de archivos offline
                         self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
                     else:
                         self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
                     if not(self.filenameList):
                         print("There is no files into the folder: %s"%(path))
                         sys.exit(-1)
                     self.fileIndex = -1
                     self.readNextFile(online)
                     '''
                     Add code
                     '''
                     self.isConfig = True
                     pass
                 def readAMISRHeader(self,fp):
                     header = 'Raw11/Data/RadacHeader'
                     self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
                     self.beamCode = fp.get('Raw11/Data/Beamcodes') # NUMBER OF CHANNELS AND IDENTIFY POSITION TO CREATE A FILE WITH THAT INFO
                     #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
                     self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
                     self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
                     self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
                     self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
                     self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
                     self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
                     self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
                     self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
                     self.frequency =  fp.get('Rx/Frequency')
                     txAus = fp.get('Raw11/Data/Pulsewidth')
                     self.nblocks = self.pulseCount.shape[0] #nblocks
                     self.nprofiles = self.pulseCount.shape[1] #nprofile
                     self.nsa = self.nsamplesPulse[0,0] #ngates
                     self.nchannels = self.beamCode.shape[1]
                     self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
                     #self.__waitForNewFile = self.nblocks  # wait depending on the number of blocks since each block is 1 sec
                     self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
                     #filling radar controller header parameters
                     self.__ippKm = self.ippSeconds *.15*1e6 # in km
                     self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
                     self.__txB = 0
                     nWindows=1
                     self.__nSamples = self.nsa
                     self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
                     self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
                     #for now until understand why the code saved is different (code included even though code not in tuf file)
                     #self.__codeType = 0
                    # self.__nCode = None
                    # self.__nBaud = None
                     self.__code = self.code
                     self.__codeType = 0
                     if self.code != None:
                         self.__codeType = 1
                     self.__nCode = self.nCode
                     self.__nBaud = self.nBaud
                     #self.__code = 0
                     #filling system header parameters
                     self.__nSamples = self.nsa
                     self.newProfiles = self.nprofiles/self.nchannels
                     self.__channelList = list(range(self.nchannels))
                     self.__frequency = self.frequency[0][0]
                 def createBuffers(self):
                     pass
                 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.walk = walk
                 def __checkPath(self):
                     if os.path.exists(self.path):
                         self.status = 1
                     else:
                         self.status = 0
                         print('Path:%s does not exists'%self.path)
                     return
                 def __selDates(self, amisr_dirname_format):
                     try:
                         year = int(amisr_dirname_format[0:4])
                         month = int(amisr_dirname_format[4:6])
                         dom = int(amisr_dirname_format[6:8])
                         thisDate = datetime.date(year,month,dom)
                         if (thisDate>=self.startDate and thisDate <= self.endDate):
                             return amisr_dirname_format
                     except:
                         return None
                 def __findDataForDates(self,online=False):
                     if not(self.status):
                         return None
                     pat = '\d+.\d+'
                     dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
                     dirnameList = [x for x in dirnameList if x!=None]
                     dirnameList = [x.string for x in dirnameList]
                     if not(online):
                         dirnameList = [self.__selDates(x) for x in dirnameList]
                         dirnameList = [x for x in dirnameList if x!=None]
                     if len(dirnameList)>0:
                         self.status = 1
                         self.dirnameList = dirnameList
                         self.dirnameList.sort()
                     else:
                         self.status = 0
                         return None
                 def __getTimeFromData(self):
                     startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
                     endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                     print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
                     print('........................................')
                     filter_filenameList = []
                     self.filenameList.sort()
                     #for i in range(len(self.filenameList)-1):
                     for i in range(len(self.filenameList)):
                         filename = self.filenameList[i]
                         fp = h5py.File(filename,'r')
                         time_str = fp.get('Time/RadacTimeString')
                         startDateTimeStr_File = time_str[0][0].split('.')[0]
                         junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                         startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                         endDateTimeStr_File = time_str[-1][-1].split('.')[0]
                         junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                         endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                         fp.close()
                         if self.timezone == 'lt':
                             startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
                             endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
                         if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
                             #self.filenameList.remove(filename)
                             filter_filenameList.append(filename)
                         if (endDateTime_File>=endDateTime_Reader):
                             break
                     filter_filenameList.sort()
                     self.filenameList = filter_filenameList
                     return 1
                 def __filterByGlob1(self, dirName):
                     filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
                     filter_files.sort()
                     filterDict = {}
                     filterDict.setdefault(dirName)
                     filterDict[dirName] = filter_files
                     return filterDict
                 def __getFilenameList(self, fileListInKeys, dirList):
                     for value in fileListInKeys:
                         dirName = list(value.keys())[0]
                         for file in value[dirName]:
                             filename = os.path.join(dirName, file)
                             self.filenameList.append(filename)
                 def __selectDataForTimes(self, online=False):
                     #aun no esta implementado el filtro for tiempo
                     if not(self.status):
                         return None
                     dirList = [os.path.join(self.path,x) for x in self.dirnameList]
                     fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
                     self.__getFilenameList(fileListInKeys, dirList)
                     if not(online):
                         #filtro por tiempo
                         if not(self.all):
                             self.__getTimeFromData()
                         if len(self.filenameList)>0:
                             self.status = 1
                             self.filenameList.sort()
                         else:
                             self.status = 0
                             return None
                     else:
                         #get the last file - 1
                         self.filenameList = [self.filenameList[-2]]
                     new_dirnameList = []
                     for dirname in self.dirnameList:
                         junk = numpy.array([dirname in x for x in self.filenameList])
                         junk_sum = junk.sum()
                         if junk_sum > 0:
                             new_dirnameList.append(dirname)
                     self.dirnameList = new_dirnameList
                     return 1
                 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),walk=True):
                     if endDate ==None:
                      startDate = datetime.datetime.utcnow().date()
                      endDate = datetime.datetime.utcnow().date()
                     self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
                     self.__checkPath()
                     self.__findDataForDates(online=True)
                     self.dirnameList = [self.dirnameList[-1]]
                     self.__selectDataForTimes(online=True)
                     return
                 def searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         walk=True):
                     self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
                     self.__checkPath()
                     self.__findDataForDates()
                     self.__selectDataForTimes()
                     for i in range(len(self.filenameList)):
                         print("%s" %(self.filenameList[i]))
                     return
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print("No more Files")
                             return 0
                         filename = self.filenameList[idFile]
                         amisrFilePointer = h5py.File(filename,'r')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.amisrFilePointer = amisrFilePointer
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def __setNextFileOnline(self):
                     filename = self.filenameList[0]
                     if self.__filename_online != None:
                         self.__selectDataForTimes(online=True)
                         filename = self.filenameList[0]
                         wait = 0
                         while self.__filename_online == filename:
                             print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
                             if wait == 5:
                                 return 0
                             sleep(self.__waitForNewFile)
                             self.__selectDataForTimes(online=True)
                             filename = self.filenameList[0]
                             wait += 1
                     self.__filename_online = filename
                     self.amisrFilePointer = h5py.File(filename,'r')
                     self.flagIsNewFile = 1
                     self.filename = filename
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def readData(self):
                     buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
                     re = buffer[:,:,:,0]
                     im = buffer[:,:,:,1]
                     dataset = re + im*1j
                     self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
                     timeset = self.radacTime[:,0]
                     return dataset,timeset
                 def reshapeData(self):
                 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
                     channels = self.beamCodeByPulse[0,:]
                     nchan = self.nchannels
                     #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
                     nblocks = self.nblocks
                     nsamples = self.nsa
                     #Dimensions : nChannels, nProfiles, nSamples
                     new_block = numpy.empty((nblocks, nchan, self.newProfiles, nsamples), dtype="complex64")
                     ############################################
                     for thisChannel in range(nchan):
                         new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[0][thisChannel])[0],:]
                     new_block = numpy.transpose(new_block, (1,0,2,3))
                     new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
                     return new_block
                 def updateIndexes(self):
                     pass
                 def fillJROHeader(self):
                     #fill radar controller header
                     self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
                                                                                   txA=self.__txA,
                                                                                   txB=0,
                                                                                   nWindows=1,
                                                                                   nHeights=self.__nSamples,
                                                                                   firstHeight=self.__firstHeight,
                                                                                   deltaHeight=self.__deltaHeight,
                                                                                   codeType=self.__codeType,
                                                                                   nCode=self.__nCode, nBaud=self.__nBaud,
                                                                                   code = self.__code,
                                                                                   fClock=1)
                     #fill system header
                     self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                 nProfiles=self.newProfiles,
                                                                 nChannels=len(self.__channelList),
                                                                 adcResolution=14,
                                                                 pciDioBusWith=32)
                     self.dataOut.type = "Voltage"
                     self.dataOut.data = None
                     self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
             #        self.dataOut.nChannels = 0
             #        self.dataOut.nHeights = 0
                     self.dataOut.nProfiles = self.newProfiles*self.nblocks
                     #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
                     ranges = numpy.reshape(self.rangeFromFile.value,(-1))
                     self.dataOut.heightList =  ranges/1000.0 #km
                     self.dataOut.channelList = self.__channelList
                     self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
             #        self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData = True
                     #Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime = None
                     #self.dataOut.timeZone = -5 #self.__timezone/60  #timezone like jroheader, difference in minutes between UTC and localtime
                     if self.timezone == 'lt':
                         self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
                     else:
                         self.dataOut.timeZone = 0 #by default time is UTC
                     self.dataOut.dstFlag = 0
                     self.dataOut.errorCount = 0
                     self.dataOut.nCohInt = 1
                     self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
                     self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.ippSeconds = self.ippSeconds
                     #Time interval between profiles
                     #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.__frequency
                     self.dataOut.realtime = self.online
                     pass
                 def readNextFile(self,online=False):
                     if not(online):
                         newFile = self.__setNextFileOffline()
                     else:
                         newFile = self.__setNextFileOnline()
                     if not(newFile):
                         return 0
                     #if self.__firstFile:
                     self.readAMISRHeader(self.amisrFilePointer)
                     self.createBuffers()
                     self.fillJROHeader()
                     #self.__firstFile = False
                     self.dataset,self.timeset = self.readData()
                     if self.endDate!=None:
                      endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                      time_str = self.amisrFilePointer.get('Time/RadacTimeString')
                      startDateTimeStr_File = time_str[0][0].split('.')[0]
                      junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                      startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                      if self.timezone == 'lt':
                       startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
                      if (startDateTime_File>endDateTime_Reader):
                          return 0
                     self.jrodataset = self.reshapeData()
                     #----self.updateIndexes()
                     self.profileIndex = 0
                     return 1
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= (self.newProfiles*self.nblocks):
                         return 1
                     return 0
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
-                        print('Process finished')
                         return 0
                     if self.__hasNotDataInBuffer():
                         if not (self.readNextFile(self.online)):
                             return 0
                     if self.dataset is None: # setear esta condicion cuando no hayan datos por leers
                         self.dataOut.flagNoData = True
                         return 0
                     #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
                     self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
                     #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
                     #verificar basic header de jro data y ver si es compatible con este valor
                     #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
                     indexprof = numpy.mod(self.profileIndex, self.newProfiles)
                     indexblock = self.profileIndex/self.newProfiles
                     #print indexblock, indexprof
                     self.dataOut.utctime = self.timeset[indexblock] + (indexprof * self.ippSeconds * self.nchannels)
                     self.dataOut.profileIndex = self.profileIndex
                     self.dataOut.flagNoData = False
             #         if indexprof == 0:
             #             print self.dataOut.utctime
                     self.profileIndex += 1
                     return self.dataOut.data
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             class Writer(Operation):
                 '''
                 classdocs
                 '''
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     self.dataOut = None
                     self.isConfig = False
                 def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
                     '''
                     In this method we should set all initial parameters.
                     Input:
                         dataIn        :        Input data will also be outputa data
                     '''
                     self.dataOut = dataIn
                     self.isConfig = True
                     return
                 def run(self, dataIn, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     Inputs:
                         dataIn        :        object with the data
                     '''
                     if not self.isConfig:
                         self.setup(dataIn, **kwargs)
   No newline at end of file

schainpy/model/io/jroIO_madrigal.py +3 -1

             '''
             Created on Aug 1, 2017
             @author: Juan C. Espinoza
             '''
             import os
             import sys
             import time
             import json
             import glob
             import datetime
             import numpy
             import h5py
+            import schainpy.admin
             from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jrodata import Parameters
             from schainpy.utils import log
             try:
                 import madrigal.cedar
             except:
                 log.warning(
                     'You should install "madrigal library" module if you want to read/write Madrigal data'
                     )
             try:
                 basestring
             except:
                 basestring = str
             DEF_CATALOG = {
                 'principleInvestigator': 'Marco Milla',
                 'expPurpose': '',
                 'cycleTime': '',
                 'correlativeExp': '',
                 'sciRemarks': '',
                 'instRemarks': ''
                 }
             DEF_HEADER = {
                 'kindatDesc': '',
                 'analyst': 'Jicamarca User',
                 'comments': '',
                 'history': ''
                 }
             MNEMONICS = {
 : 'jro',
 : 'jbr',
 : 'jul',
 : 'jas',
 : 'pbr',
 : 'hbr',
 : 'obr',
 : 'clr'
             }
             UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
             def load_json(obj):
                 '''
                 Parse json as string instead of unicode
                 '''
                 if isinstance(obj, str):
                     iterable = json.loads(obj)
                 else:
                     iterable = obj
                 if isinstance(iterable, dict):
                     return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, basestring) else v
                         for k, v in list(iterable.items())}
                 elif isinstance(iterable, (list, tuple)):
                     return [str(v) if isinstance(v, basestring) else v for v in iterable]
                 return iterable
             @MPDecorator
             class MADReader(JRODataReader, ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                     self.counter_records = 0
                     self.nrecords = None
                     self.flagNoMoreFiles = 0
                     self.isConfig = False
                     self.filename = None
                     self.intervals = set()
                 def setup(self,
                           path=None,
                           startDate=None,
                           endDate=None,
                           format=None,
                           startTime=datetime.time(0, 0, 0),
                           endTime=datetime.time(23, 59, 59),
                           **kwargs):
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.datatime = datetime.datetime(1900,1,1)
                     self.oneDDict = load_json(kwargs.get('oneDDict',
                                                          "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
                     self.twoDDict = load_json(kwargs.get('twoDDict',
                                                          "{\"GDALT\": \"heightList\"}"))
                     self.independentParam = 'GDALT'
                     if self.path is None:
                         raise ValueError('The path is not valid')
                     if format is None:
                         raise ValueError('The format is not valid choose simple or hdf5')
                     elif format.lower() in ('simple', 'txt'):
                         self.ext = '.txt'
                     elif format.lower() in ('cedar',):
                         self.ext = '.001'
                     else:
                         self.ext = '.hdf5'
                     self.search_files(self.path)
                     self.fileId = 0
                     if not self.fileList:
                         raise  Warning('There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path))
                     self.setNextFile()
                 def search_files(self, path):
                     '''
                      Searching for madrigal files in path
                      Creating a list of files to procces included in [startDate,endDate]
                      Input:
                          path - Path to find files
                     '''
                     log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
                     fileList0 = glob.glob1(path, '*{}'.format(self.ext))
                     fileList0.sort()
                     self.fileList = []
                     self.dateFileList = []
                     startDate = self.startDate - datetime.timedelta(1)
                     endDate = self.endDate + datetime.timedelta(1)
                     for thisFile in fileList0:
                         year = thisFile[3:7]
                         if not year.isdigit():
                             continue
                         month = thisFile[7:9]
                         if not month.isdigit():
                             continue
                         day = thisFile[9:11]
                         if not day.isdigit():
                             continue
                         year, month, day = int(year), int(month), int(day)
                         dateFile = datetime.date(year, month, day)
                         if (startDate > dateFile) or (endDate < dateFile):
                             continue
                         self.fileList.append(thisFile)
                         self.dateFileList.append(dateFile)
                     return
                 def parseHeader(self):
                     '''
                     '''
                     self.output = {}
                     self.version = '2'
                     s_parameters = None
                     if self.ext == '.txt':
                         self.parameters = [s.strip().lower() for s in self.fp.readline().decode().strip().split(' ') if s]
                     elif self.ext == '.hdf5':
                         self.metadata = self.fp['Metadata']
                         if '_record_layout' in self.metadata:
                             s_parameters = [s[0].lower().decode() for s in self.metadata['Independent Spatial Parameters']]
                             self.version = '3'
                         self.parameters = [s[0].lower().decode() for s in self.metadata['Data Parameters']]
                     log.success('Parameters found: {}'.format(self.parameters),
                                 'MADReader')
                     if s_parameters:
                         log.success('Spatial parameters found: {}'.format(s_parameters),
                                     'MADReader')
                     for param in list(self.oneDDict.keys()):
                         if param.lower() not in self.parameters:
                             log.warning(
                                 'Parameter {} not found will be ignored'.format(
                                     param),
                                 'MADReader')
                             self.oneDDict.pop(param, None)
                     for param, value in list(self.twoDDict.items()):
                         if param.lower() not in self.parameters:
                             log.warning(
                                 'Parameter {} not found, it will be ignored'.format(
                                     param),
                                 'MADReader')
                             self.twoDDict.pop(param, None)
                             continue
                         if isinstance(value, list):
                             if value[0] not in self.output:
                                 self.output[value[0]] = []
                             self.output[value[0]].append([])
                 def parseData(self):
                     '''
                     '''
                     if self.ext == '.txt':
                         self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
                         self.nrecords = self.data.shape[0]
                         self.ranges = numpy.unique(self.data[:,self.parameters.index(self.independentParam.lower())])
                         self.counter_records = 0
                     elif self.ext == '.hdf5':
                         self.data = self.fp['Data']
                         self.ranges = numpy.unique(self.data['Table Layout'][self.independentParam.lower()])
                         self.times = numpy.unique(self.data['Table Layout']['ut1_unix'])
                         self.counter_records = int(self.data['Table Layout']['recno'][0])
                         self.nrecords = int(self.data['Table Layout']['recno'][-1])
                 def setNextFile(self):
                     '''
                     '''
                     file_id = self.fileId
                     if file_id == len(self.fileList):
                         log.success('No more files', 'MADReader')
                         self.flagNoMoreFiles = 1
                         return 0
                     log.success(
                         'Opening: {}'.format(self.fileList[file_id]),
                         'MADReader'
                         )
                     filename = os.path.join(self.path, self.fileList[file_id])
                     if self.filename is not None:
                         self.fp.close()
                     self.filename = filename
                     self.filedate = self.dateFileList[file_id]
                     if self.ext=='.hdf5':
                         self.fp = h5py.File(self.filename, 'r')
                     else:
                         self.fp = open(self.filename, 'rb')
                     self.parseHeader()
                     self.parseData()
                     self.sizeOfFile = os.path.getsize(self.filename)
                     self.flagIsNewFile = 0
                     self.fileId += 1
                     return 1
                 def readNextBlock(self):
                     while True:
                         self.flagDiscontinuousBlock = 0
                         if self.flagIsNewFile:
                             if not self.setNextFile():
                                 return 0
                         self.readBlock()
                         if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
                            (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
                             log.warning(
                                 'Reading Record No. {}/{} -> {} [Skipping]'.format(
                                     self.counter_records,
                                     self.nrecords,
                                     self.datatime.ctime()),
                                 'MADReader')
                             continue
                         break
                     log.log(
                         'Reading Record No. {}/{} -> {}'.format(
                             self.counter_records,
                             self.nrecords,
                             self.datatime.ctime()),
                         'MADReader')
                     return 1
                 def readBlock(self):
                     '''
                     '''
                     dum = []
                     if self.ext == '.txt':
                         dt = self.data[self.counter_records][:6].astype(int)
                         if datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]).date() > self.datatime.date():
                             self.flagDiscontinuousBlock = 1
                         self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
                         while True:
                             dt = self.data[self.counter_records][:6].astype(int)
                             datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
                             if datatime == self.datatime:
                                 dum.append(self.data[self.counter_records])
                                 self.counter_records += 1
                                 if self.counter_records == self.nrecords:
                                     self.flagIsNewFile = True
                                     break
                                 continue
                             self.intervals.add((datatime-self.datatime).seconds)
                             break
                     elif self.ext == '.hdf5':
                         datatime = datetime.datetime.utcfromtimestamp(
                             self.times[self.counter_records])
                         dum = self.data['Table Layout'][self.data['Table Layout']['recno']==self.counter_records]
                         self.intervals.add((datatime-self.datatime).seconds)
                         if datatime.date()>self.datatime.date():
                             self.flagDiscontinuousBlock = 1
                         self.datatime = datatime
                         self.counter_records += 1
                         if self.counter_records == self.nrecords:
                             self.flagIsNewFile = True
                     self.buffer = numpy.array(dum)
                     return
                 def set_output(self):
                     '''
                     Storing data from buffer to dataOut object
                     '''
                     parameters = [None for __ in self.parameters]
                     for param, attr in list(self.oneDDict.items()):
                         x = self.parameters.index(param.lower())
                         setattr(self.dataOut, attr, self.buffer[0][x])
                     for param, value in list(self.twoDDict.items()):
                         dummy = numpy.zeros(self.ranges.shape) + numpy.nan
                         if self.ext == '.txt':
                             x = self.parameters.index(param.lower())
                             y = self.parameters.index(self.independentParam.lower())
                             ranges = self.buffer[:,y]
                             #if self.ranges.size == ranges.size:
                             #    continue
                             index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
                             dummy[index] = self.buffer[:,x]
                         else:
                             ranges = self.buffer[self.independentParam.lower()]
                             index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
                             dummy[index] = self.buffer[param.lower()]
                         if isinstance(value, str):
                             if value not in self.independentParam:
                                 setattr(self.dataOut, value, dummy.reshape(1,-1))
                         elif isinstance(value, list):
                             self.output[value[0]][value[1]] = dummy
                             parameters[value[1]] = param
                     for key, value in list(self.output.items()):
                         setattr(self.dataOut, key, numpy.array(value))
                     self.dataOut.parameters = [s for s in parameters if s]
                     self.dataOut.heightList = self.ranges
                     self.dataOut.utctime = (self.datatime - datetime.datetime(1970, 1, 1)).total_seconds()
                     self.dataOut.utctimeInit = self.dataOut.utctime
                     self.dataOut.paramInterval = min(self.intervals)
                     self.dataOut.useLocalTime = False
                     self.dataOut.flagNoData = False
                     self.dataOut.nrecords = self.nrecords
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                 def getData(self):
                     '''
                     Storing data from databuffer to dataOut object
                     '''
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
-                        self.dataOut.error = 'No file left to process'
+                        raise schainpy.admin.SchainError('No file left to process')
                         return 0
                     if not  self.readNextBlock():
                         self.dataOut.flagNoData = True
                         return 0
                     self.set_output()
                     return 1
             @MPDecorator
             class MADWriter(Operation):
                 missing = -32767
                 def __init__(self):
                     Operation.__init__(self)
                     self.dataOut = Parameters()
                     self.counter = 0
                     self.path = None
                     self.fp = None
                 def run(self, dataOut, path, oneDDict, independentParam='[]', twoDDict='{}',
                         metadata='{}', format='cedar', **kwargs):
                     '''
                     Inputs:
                         path - path where files will be created
                         oneDDict - json of one-dimensional parameters in record where keys
                         are Madrigal codes (integers or mnemonics) and values the corresponding
                         dataOut attribute e.g: {
                             'gdlatr': 'lat',
                             'gdlonr': 'lon',
                             'gdlat2':'lat',
                             'glon2':'lon'}
                         independentParam - list of independent spatial two-dimensional parameters e.g:
                             ['heigthList']
                         twoDDict - json of two-dimensional parameters in record where keys
                         are Madrigal codes (integers or mnemonics) and values the corresponding
                         dataOut attribute if multidimensional array specify as tupple
                         ('attr', pos) e.g: {
                             'gdalt': 'heightList',
                             'vn1p2': ('data_output', 0),
                             'vn2p2': ('data_output', 1),
                             'vn3': ('data_output', 2),
                             'snl': ('data_SNR', 'db')
                             }
                         metadata - json of madrigal metadata (kinst, kindat, catalog and header)
                     '''
                     if not self.isConfig:
                         self.setup(path, oneDDict, independentParam, twoDDict, metadata, format, **kwargs)
                         self.isConfig = True
                     self.dataOut = dataOut
                     self.putData()
                     return 1
                 def setup(self, path, oneDDict, independentParam, twoDDict, metadata, format, **kwargs):
                     '''
                     Configure Operation
                     '''
                     self.path = path
                     self.blocks = kwargs.get('blocks', None)
                     self.counter = 0
                     self.oneDDict = load_json(oneDDict)
                     self.twoDDict = load_json(twoDDict)
                     self.independentParam = load_json(independentParam)
                     meta = load_json(metadata)
                     self.kinst = meta.get('kinst')
                     self.kindat = meta.get('kindat')
                     self.catalog = meta.get('catalog', DEF_CATALOG)
                     self.header = meta.get('header', DEF_HEADER)
                     if format == 'cedar':
                         self.ext = '.dat'
                         self.extra_args = {}
                     elif format == 'hdf5':
                         self.ext = '.hdf5'
                         self.extra_args = {'independentParam': self.independentParam}
                     self.keys = [k.lower() for k in self.twoDDict]
                     if 'range' in self.keys:
                         self.keys.remove('range')
                     if 'gdalt' in self.keys:
                         self.keys.remove('gdalt')
                 def setFile(self):
                     '''
                     Create new cedar file object
                     '''
                     self.mnemonic = MNEMONICS[self.kinst]   #TODO get mnemonic from madrigal
                     date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
                     filename = '{}{}{}'.format(self.mnemonic,
                                                date.strftime('%Y%m%d_%H%M%S'),
                                                self.ext)
                     self.fullname = os.path.join(self.path, filename)
                     if os.path.isfile(self.fullname) :
                         log.warning(
                             'Destination file {} already exists, previous file deleted.'.format(
                                 self.fullname),
                             'MADWriter')
                         os.remove(self.fullname)
                     try:
                         log.success(
                             'Creating file: {}'.format(self.fullname),
                             'MADWriter')
                         self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
                     except ValueError as e:
                         log.error(
                             'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
                             'MADWriter')
                         return
                     return 1
                 def writeBlock(self):
                     '''
                     Add data records to cedar file taking data from oneDDict and twoDDict
                     attributes.
                     Allowed parameters in: parcodes.tab
                     '''
                     startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
                     endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
                     heights = self.dataOut.heightList
                     if self.ext == '.dat':
                         for key, value in list(self.twoDDict.items()):
                             if isinstance(value, str):
                                 data = getattr(self.dataOut, value)
                                 invalid = numpy.isnan(data)
                                 data[invalid] = self.missing
                             elif isinstance(value, (tuple, list)):
                                 attr, key = value
                                 data = getattr(self.dataOut, attr)
                                 invalid = numpy.isnan(data)
                                 data[invalid] = self.missing
                     out = {}
                     for key, value in list(self.twoDDict.items()):
                         key = key.lower()
                         if isinstance(value, str):
                             if 'db' in value.lower():
                                 tmp = getattr(self.dataOut, value.replace('_db', ''))
                                 SNRavg = numpy.average(tmp, axis=0)
                                 tmp = 10*numpy.log10(SNRavg)
                             else:
                                 tmp = getattr(self.dataOut, value)
                             out[key] = tmp.flatten()
                         elif isinstance(value, (tuple, list)):
                             attr, x = value
                             data = getattr(self.dataOut, attr)
                             out[key] = data[int(x)]
                     a = numpy.array([out[k] for k in self.keys])
                     nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
                     index = numpy.where(nrows == False)[0]
                     rec = madrigal.cedar.MadrigalDataRecord(
                         self.kinst,
                         self.kindat,
                         startTime.year,
                         startTime.month,
                         startTime.day,
                         startTime.hour,
                         startTime.minute,
                         startTime.second,
                         startTime.microsecond/10000,
                         endTime.year,
                         endTime.month,
                         endTime.day,
                         endTime.hour,
                         endTime.minute,
                         endTime.second,
                         endTime.microsecond/10000,
                         list(self.oneDDict.keys()),
                         list(self.twoDDict.keys()),
                         len(index),
                         **self.extra_args
                     )
                     # Setting 1d values
                     for key in self.oneDDict:
                         rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
                     # Setting 2d values
                     nrec = 0
                     for n in index:
                         for key in out:
                             rec.set2D(key, nrec, out[key][n])
                         nrec += 1
                     self.fp.append(rec)
                     if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
                         self.fp.dump()
                     if self.counter % 20 == 0 and self.counter > 0:
                         log.log(
                             'Writing {} records'.format(
                                 self.counter),
                             'MADWriter')
                 def setHeader(self):
                     '''
                     Create an add catalog and header to cedar file
                     '''
                     log.success('Closing file {}'.format(self.fullname), 'MADWriter')
                     if self.ext == '.dat':
                         self.fp.write()
                     else:
                         self.fp.dump()
                         self.fp.close()
                     header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
                     header.createCatalog(**self.catalog)
                     header.createHeader(**self.header)
                     header.write()
                 def putData(self):
                     if self.dataOut.flagNoData:
                         return 0
                     if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
                         if self.counter > 0:
                             self.setHeader()
                         self.counter = 0
                     if self.counter == 0:
                         self.setFile()
                     self.writeBlock()
                     self.counter += 1
                 def close(self):
                     if self.counter > 0:
                         self.setHeader()

schainpy/model/io/jroIO_param.py +3 -3

             import numpy
             import time
             import os
             import h5py
             import re
             import datetime
+            import schainpy.admin
             from schainpy.model.data.jrodata import *
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.io.jroIO_base import *
             from schainpy.utils import log
             @MPDecorator
             class ParamReader(JRODataReader,ProcessingUnit):
                 '''
                 Reads HDF5 format files
                 path
                 startDate
                 endDate
                 startTime
                 endTime
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 timezone = None
                 startTime = None
                 endTime = None
                 fileIndex = None
                 utcList = None      #To select data in the utctime list
                 blockList = None    #List to blocks to be read from the file
                 blocksPerFile = None    #Number of blocks to be read
                 blockIndex = None
                 path = None
                 #List of Files
                 filenameList = None
                 datetimeList = None
                 #Hdf5 File
                 listMetaname = None
                 listMeta = None
                 listDataname = None
                 listData = None
                 listShapes = None
                 fp = None
                 #dataOut reconstruction
                 dataOut = None
                 def __init__(self):#, **kwargs):
                     ProcessingUnit.__init__(self) #, **kwargs)
                     self.dataOut = Parameters()
                     return
                 def setup(self, **kwargs):
                     path = kwargs['path']
                     startDate = kwargs['startDate']
                     endDate = kwargs['endDate']
                     startTime = kwargs['startTime']
                     endTime = kwargs['endTime']
                     walk = kwargs['walk']
                     if 'ext' in kwargs:
                         ext = kwargs['ext']
                     else:
                         ext = '.hdf5'
                     if 'timezone' in kwargs:
                         self.timezone = kwargs['timezone']
                     else:
                         self.timezone = 'lt'
                     print("[Reading] Searching files in offline mode ...")
                     pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            ext=ext, walk=walk)
                     if not(filenameList):
                         print("There is no files into the folder: %s"%(path))
                         sys.exit(-1)
                     self.fileIndex = -1
                     self.startTime = startTime
                     self.endTime = endTime
                     self.__readMetadata()
                     self.__setNextFileOffline()
                     return
                 def searchFilesOffLine(self,
                                         path,
                                         startDate=None,
                                         endDate=None,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         ext='.hdf5',
                                         walk=True):
                     expLabel = ''
                     self.filenameList = []
                     self.datetimeList = []
                     pathList = []
                     JRODataObj = JRODataReader()
                     dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
                     if dateList == []:
                         print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime()))
                         return None, None
                     if len(dateList) > 1:
                         print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
                     else:
                         print("[Reading] data was found for the date %s" %(dateList[0]))
                     filenameList = []
                     datetimeList = []
                     #----------------------------------------------------------------------------------
                     for thisPath in pathList:
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             if not isFileInDateRange(filename, startDate, endDate):
                                 continue
                             thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
                         return None, None
                     print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
                     print()
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
                     """
                     Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                     Inputs:
                         filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                         startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                         endDate            :    fecha final del rango seleccionado en formato datetime.date
                         startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                         endTime            :    tiempo final del rango seleccionado en formato datetime.time
                     Return:
                         Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                         fecha especificado, de lo contrario retorna False.
                     Excepciones:
                         Si el archivo no existe o no puede ser abierto
                         Si la cabecera no puede ser leida.
                     """
                     try:
                         fp = h5py.File(filename,'r')
                         grp1 = fp['Data']
                     except IOError:
                         traceback.print_exc()
                         raise IOError("The file %s can't be opened" %(filename))
                     #In case has utctime attribute
                     grp2 = grp1['utctime']
              #         thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
                     thisUtcTime = grp2.value[0]
                     fp.close()
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
                     #General case
                     #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                     #-----------o----------------------------o-----------
                     #       startTime                     endTime
                     if endTime >= startTime:
                         thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
                         if numpy.any(thisUtcLog):   #If there is one block between the hours mentioned
                             return thisDatetime
                         return None
                     #If endTime < startTime then endTime belongs to the next day
                     #<<<<<<<<<<<o                            o>>>>>>>>>>>
                     #-----------o----------------------------o-----------
                     #        endTime                    startTime
                     if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
                         return None
                     if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
                         return None
                     if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
                         return None
                     return thisDatetime
                 def __setNextFileOffline(self):
                     self.fileIndex += 1
                     idFile = self.fileIndex
                     if not(idFile < len(self.filenameList)):
-                        self.dataOut.error = "No more Files"
+                        raise schainpy.admin.SchainError("No more Files")
                         return 0
                     filename = self.filenameList[idFile]
                     filePointer = h5py.File(filename,'r')
                     self.filename = filename
                     self.fp = filePointer
                     print("Setting the file: %s"%self.filename)
                     self.__setBlockList()
                     self.__readData()
                     self.blockIndex = 0
                     return 1
                 def __setBlockList(self):
                     '''
                     Selects the data within the times defined
                     self.fp
                     self.startTime
                     self.endTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     fp = self.fp
                     startTime = self.startTime
                     endTime = self.endTime
                     grp = fp['Data']
                     thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
                     #ERROOOOR
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     return
                 def __readMetadata(self):
                     '''
                     Reads Metadata
                     self.pathMeta
                     self.listShapes
                     self.listMetaname
                     self.listMeta
                     '''
                     filename = self.filenameList[0]
                     fp = h5py.File(filename,'r')
                     gp = fp['Metadata']
                     listMetaname = []
                     listMetadata = []
                     for item in list(gp.items()):
                         name = item[0]
                         if name=='array dimensions':
                             table = gp[name][:]
                             listShapes = {}
                             for shapes in table:
                                 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
                         else:
                             data = gp[name].value
                             listMetaname.append(name)
                             listMetadata.append(data)
                     self.listShapes = listShapes
                     self.listMetaname = listMetaname
                     self.listMeta = listMetadata
                     fp.close()
                     return
                 def __readData(self):
                     grp = self.fp['Data']
                     listdataname = []
                     listdata = []
                     for item in list(grp.items()):
                         name = item[0]
                         listdataname.append(name)
                         array = self.__setDataArray(grp[name],self.listShapes[name])
                         listdata.append(array)
                     self.listDataname = listdataname
                     self.listData = listdata
                     return
                 def __setDataArray(self, dataset, shapes):
                     nDims = shapes[0]
                     nDim2 = shapes[1]      #Dimension 0
                     nDim1 = shapes[2]      #Dimension 1, number of Points or Parameters
                     nDim0 = shapes[3]      #Dimension 2, number of samples or ranges
                     mode = shapes[4]        #Mode of storing
                     blockList = self.blockList
                     blocksPerFile = self.blocksPerFile
                     #Depending on what mode the data was stored
                     if mode == 0:       #Divided in channels
                         arrayData = dataset.value.astype(numpy.float)[0][blockList]
                     if mode == 1:     #Divided in parameter
                         strds = 'table'
                         nDatas = nDim1
                         newShapes = (blocksPerFile,nDim2,nDim0)
                     elif mode==2:       #Concatenated in a table
                         strds = 'table0'
                         arrayData = dataset[strds].value
                         #Selecting part of the dataset
                         utctime = arrayData[:,0]
                         u, indices = numpy.unique(utctime, return_index=True)
                         if blockList.size != indices.size:
                             indMin = indices[blockList[0]]
                             if blockList[1] + 1 >= indices.size:
                                 arrayData = arrayData[indMin:,:]
                             else:
                                 indMax = indices[blockList[1] + 1]
                                 arrayData = arrayData[indMin:indMax,:]
                         return arrayData
                     #    One dimension
                     if nDims == 0:
                         arrayData = dataset.value.astype(numpy.float)[0][blockList]
                     #    Two dimensions
                     elif nDims == 2:
                         arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
                         newShapes = (blocksPerFile,nDim0)
                         nDatas = nDim1
                         for i in range(nDatas):
                             data = dataset[strds + str(i)].value
                             arrayData[:,i,:] = data[blockList,:]
                     #    Three dimensions
                     else:
                         arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
                         for i in range(nDatas):
                             data = dataset[strds + str(i)].value
                             for b in range(blockList.size):
                                 arrayData[b,:,i,:] = data[:,:,blockList[b]]
                     return arrayData
                 def __setDataOut(self):
                     listMeta = self.listMeta
                     listMetaname = self.listMetaname
                     listDataname = self.listDataname
                     listData = self.listData
                     listShapes = self.listShapes
                     blockIndex = self.blockIndex
              #         blockList = self.blockList
                     for i in range(len(listMeta)):
                         setattr(self.dataOut,listMetaname[i],listMeta[i])
                     for j in range(len(listData)):
                         nShapes = listShapes[listDataname[j]][0]
                         mode = listShapes[listDataname[j]][4]
                         if nShapes == 1:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
                         elif nShapes > 1:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
                         elif mode==0:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
                         #Mode Meteors
                         elif mode ==2:
                             selectedData = self.__selectDataMode2(listData[j], blockIndex)
                             setattr(self.dataOut, listDataname[j], selectedData)
                     return
                 def __selectDataMode2(self, data, blockIndex):
                     utctime = data[:,0]
                     aux, indices = numpy.unique(utctime, return_inverse=True)
                     selInd = numpy.where(indices == blockIndex)[0]
                     selData = data[selInd,:]
                     return selData
                 def getData(self):
                     if self.blockIndex==self.blocksPerFile:
                          if not( self.__setNextFileOffline() ):
                             self.dataOut.flagNoData = True
                             return 0
                     self.__setDataOut()
                     self.dataOut.flagNoData = False
                     self.blockIndex += 1
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
                     return
             @MPDecorator
             class ParamWriter(Operation):
                 '''
                 HDF5 Writer, stores parameters data in HDF5 format files
                 path:             path where the files will be stored
                 blocksPerFile:    number of blocks that will be saved in per HDF5 format file
                 mode:             selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
                 metadataList:     list of attributes that will be stored as metadata
                 dataList:         list of attributes that will be stores as data
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 metaoptchar = "M"
                 metaFile = None
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 grp = None
                 ds = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 dsList = None   #List of dictionaries with dataset properties
                 tableDim = None
                 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
                 currentDay = None
                 lastTime = None
                 setType = None
                 def __init__(self):
                     Operation.__init__(self)
                     return
                 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataList = dataList
                     self.dataOut = dataOut
                     self.mode = mode
                     if self.mode is not None:
                         self.mode = numpy.zeros(len(self.dataList)) + mode
                     else:
                         self.mode = numpy.ones(len(self.dataList))
                     self.setType = setType
                     arrayDim = numpy.zeros((len(self.dataList),5))
                     #Table dimensions
                     dtype0 = self.dtype
                     tableList = []
                     #Dictionary and list of tables
                     dsList = []
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         dataAux = getattr(self.dataOut, self.dataList[i])
                         dsDict['variable'] = self.dataList[i]
                         #---------------------    Conditionals    ------------------------
                         #There is no data
                         if dataAux is None:
                             return 0
                         if isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
                             dsDict['mode'] = 0
                             dsDict['nDim'] = 0
                             arrayDim[i,0] = 0
                             dsList.append(dsDict)
                         #Mode 2: meteors
                         elif self.mode[i] == 2:
                             dsDict['dsName'] = 'table0'
                             dsDict['mode'] = 2      # Mode meteors
                             dsDict['shape'] = dataAux.shape[-1]
                             dsDict['nDim'] = 0
                             dsDict['dsNumber'] = 1
                             arrayDim[i,3] = dataAux.shape[-1]
                             arrayDim[i,4] = self.mode[i]         #Mode the data was stored
                             dsList.append(dsDict)
                         #Mode 1
                         else:
                             arrayDim0 = dataAux.shape       #Data dimensions
                             arrayDim[i,0] = len(arrayDim0)  #Number of array dimensions
                             arrayDim[i,4] = self.mode[i]         #Mode the data was stored
                             strtable = 'table'
                             dsDict['mode'] = 1      # Mode parameters
                             # Three-dimension arrays
                             if len(arrayDim0) == 3:
                                 arrayDim[i,1:-1] = numpy.array(arrayDim0)
                                 nTables = int(arrayDim[i,2])
                                 dsDict['dsNumber'] = nTables
                                 dsDict['shape'] = arrayDim[i,2:4]
                                 dsDict['nDim'] = 3
                                 for j in range(nTables):
                                     dsDict = dsDict.copy()
                                     dsDict['dsName'] = strtable + str(j)
                                     dsList.append(dsDict)
                             # Two-dimension arrays
                             elif len(arrayDim0) == 2:
                                 arrayDim[i,2:-1] = numpy.array(arrayDim0)
                                 nTables = int(arrayDim[i,2])
                                 dsDict['dsNumber'] = nTables
                                 dsDict['shape'] = arrayDim[i,3]
                                 dsDict['nDim'] = 2
                                 for j in range(nTables):
                                     dsDict = dsDict.copy()
                                     dsDict['dsName'] = strtable + str(j)
                                     dsList.append(dsDict)
                             # One-dimension arrays
                             elif len(arrayDim0) == 1:
                                 arrayDim[i,3] = arrayDim0[0]
                                 dsDict['shape'] = arrayDim0[0]
                                 dsDict['dsNumber'] = 1
                                 dsDict['dsName'] = strtable + str(0)
                                 dsDict['nDim'] = 1
                                 dsList.append(dsDict)
                         table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
                         tableList.append(table)
                     self.dsList = dsList
                     self.tableDim = numpy.array(tableList, dtype = dtype0)
                     self.blockIndex = 0
                     timeTuple = time.localtime(dataOut.utctime)
                     self.currentDay = timeTuple.tm_yday
                 def putMetadata(self):
                     fp = self.createMetadataFile()
                     self.writeMetadata(fp)
                     fp.close()
                     return
                 def createMetadataFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = ''
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         filesList = sorted( filesList, key=str.lower )
                         if len( filesList ) > 0:
                             filesList = [k for k in filesList if k.startswith(self.metaoptchar)]
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     filename = os.path.join( path, subfolder, file )
                     self.metaFile = file
                     #Setting HDF5 File
                     fp = h5py.File(filename,'w')
                     return fp
                 def writeMetadata(self, fp):
                     grp = fp.create_group("Metadata")
                     grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
                     for i in range(len(self.metadataList)):
                         grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
                     return
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     if self.lastTime is None:
                         self.lastTime = currentTime
                     #Day
                     timeTuple = time.localtime(currentTime)
                     dataDay = timeTuple.tm_yday
                     #Time
                     timeDiff = currentTime - self.lastTime
                     #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                     if dataDay != self.currentDay:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > 3*60*60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     mode = self.mode
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if os.path.exists(fullpath):
                         filesList = os.listdir( fullpath )
                         filesList = [k for k in filesList if 'M' in k]
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     filename = os.path.join( path, subfolder, file )
                     #Setting HDF5 File
                     fp = h5py.File(filename,'w')
                     #write metadata
                     self.writeMetadata(fp)
                     #Write data
                     grp = fp.create_group("Data")
                     ds = []
                     data = []
                     dsList = self.dsList
                     i = 0
                     while i < len(dsList):
                         dsInfo = dsList[i]
                         #One-dimension data
                         if dsInfo['mode'] == 0:
                             ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
                             ds.append(ds0)
                             data.append([])
                             i += 1
                             continue
                         elif dsInfo['mode'] == 2:
                             grp0 = grp.create_group(dsInfo['variable'])
                             ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
                             ds.append(ds0)
                             data.append([])
                             i += 1
                             continue
                         elif dsInfo['mode'] == 1:
                             grp0 = grp.create_group(dsInfo['variable'])
                             for j in range(dsInfo['dsNumber']):
                                 dsInfo = dsList[i]
                                 tableName = dsInfo['dsName']
                                 if dsInfo['nDim'] == 3:
                                     shape = dsInfo['shape'].astype(int)
                                     ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
                                 else:
                                     shape = int(dsInfo['shape'])
                                     ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
                                 ds.append(ds0)
                                 data.append([])
                                 i += 1
                     fp.flush()
                     fp.close()
                     log.log('creating file: {}'.format(filename), 'Writing')
                     self.filename = filename
                     self.ds = ds
                     self.data = data
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     if self.blockIndex == self.blocksPerFile or self.timeFlag():
                         self.setNextFile()
                     self.readBlock()
                     self.setBlock()     #Prepare data to be written
                     self.writeBlock()   #Write data
                     return
                 def readBlock(self):
                     '''
                     data Array configured
                     self.data
                     '''
                     dsList = self.dsList
                     ds = self.ds
                             #Setting HDF5 File
                     fp = h5py.File(self.filename,'r+')
                     grp = fp["Data"]
                     ind = 0
                     while ind < len(dsList):
                         dsInfo = dsList[ind]
                         if dsInfo['mode'] == 0:
                             ds0 = grp[dsInfo['variable']]
                             ds[ind] = ds0
                             ind += 1
                         else:
                             grp0 = grp[dsInfo['variable']]
                             for j in range(dsInfo['dsNumber']):
                                 dsInfo = dsList[ind]
                                 ds0 = grp0[dsInfo['dsName']]
                                 ds[ind] = ds0
                                 ind += 1
                     self.fp = fp
                     self.grp = grp
                     self.ds = ds
                     return
                 def setBlock(self):
                     '''
                     data Array configured
                     self.data
                     '''
                     #Creating Arrays
                     dsList = self.dsList
                     data = self.data
                     ind = 0
                     while ind < len(dsList):
                         dsInfo = dsList[ind]
                         dataAux = getattr(self.dataOut, dsInfo['variable'])
                         mode = dsInfo['mode']
                         nDim = dsInfo['nDim']
                         if mode == 0 or mode == 2 or nDim == 1:
                             data[ind] = dataAux
                             ind += 1
              #             elif nDim == 1:
              #                 data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
              #                 ind += 1
                         elif nDim == 2:
                             for j in range(dsInfo['dsNumber']):
                                 data[ind] = dataAux[j,:]
                                 ind += 1
                         elif nDim == 3:
                             for j in range(dsInfo['dsNumber']):
                                 data[ind] = dataAux[:,j,:]
                                 ind += 1
                     self.data = data
                     return
                 def writeBlock(self):
                     '''
                     Saves the block in the HDF5 file
                     '''
                     dsList = self.dsList
                     for i in range(len(self.ds)):
                         dsInfo = dsList[i]
                         nDim = dsInfo['nDim']
                         mode = dsInfo['mode']
                         #    First time
                         if self.firsttime:
                             if type(self.data[i]) == numpy.ndarray:
                                 if nDim == 3:
                                     self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
                                     self.ds[i].resize(self.data[i].shape)
                                 if mode == 2:
                                     self.ds[i].resize(self.data[i].shape)
                             self.ds[i][:] = self.data[i]
                         else:
                         #    From second time
                             #    Meteors!
                             if mode == 2:
                                 dataShape = self.data[i].shape
                                 dsShape = self.ds[i].shape
                                 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
                                 self.ds[i][dsShape[0]:,:] = self.data[i]
                             #    No dimension
                             elif mode == 0:
                                 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
                                 self.ds[i][0,-1] = self.data[i]
                             #    One dimension
                             elif nDim == 1:
                                 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
                                 self.ds[i][-1,:] = self.data[i]
                             #    Two dimension
                             elif nDim == 2:
                                 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
                                 self.ds[i][self.blockIndex,:] = self.data[i]
                             #    Three dimensions
                             elif nDim == 3:
                                 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
                                 self.ds[i][:,:,-1] = self.data[i]
                     self.firsttime = False
                     self.blockIndex += 1
                     #Close to save changes
                     self.fp.flush()
                     self.fp.close()
                     return
                 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
                     self.dataOut = dataOut
                     if not(self.isConfig):
                         self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList, mode=mode,
                                    setType=setType)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return
             @MPDecorator
             class ParameterReader(JRODataReader,ProcessingUnit):
                 '''
                 Reads HDF5 format files
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 timezone = None
                 startTime = None
                 endTime = None
                 fileIndex = None
                 blockList = None    #List to blocks to be read from the file
                 blocksPerFile = None    #Number of blocks to be read
                 blockIndex = None
                 path = None
                 #List of Files
                 filenameList = None
                 datetimeList = None
                 #Hdf5 File
                 listMetaname = None
                 listMeta = None
                 listDataname = None
                 listData = None
                 listShapes = None
                 fp = None
                 #dataOut reconstruction
                 dataOut = None
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                     return
                 def setup(self, **kwargs):
                     path = kwargs['path']
                     startDate = kwargs['startDate']
                     endDate = kwargs['endDate']
                     startTime = kwargs['startTime']
                     endTime = kwargs['endTime']
                     walk = kwargs['walk']
                     if 'ext' in kwargs:
                         ext = kwargs['ext']
                     else:
                         ext = '.hdf5'
                     if 'timezone' in kwargs:
                         self.timezone = kwargs['timezone']
                     else:
                         self.timezone = 'lt'
                     print("[Reading] Searching files in offline mode ...")
                     pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            ext=ext, walk=walk)
                     if not(filenameList):
                         print("There is no files into the folder: %s"%(path))
                         sys.exit(-1)
                     self.fileIndex = -1
                     self.startTime = startTime
                     self.endTime = endTime
                     self.__readMetadata()
                     self.__setNextFileOffline()
                     return
                 def searchFilesOffLine(self, path, startDate=None, endDate=None, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59), ext='.hdf5', walk=True):
                     expLabel = ''
                     self.filenameList = []
                     self.datetimeList = []
                     pathList = []
                     dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
                     if dateList == []:
                         print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime()))
                         return None, None
                     if len(dateList) > 1:
                         print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
                     else:
                         print("[Reading] data was found for the date %s" %(dateList[0]))
                     filenameList = []
                     datetimeList = []
                     for thisPath in pathList:
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             if not isFileInDateRange(filename, startDate, endDate):
                                 continue
                             thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
                         return None, None
                     print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
                     print()
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
                     """
                     Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                     Inputs:
                         filename           :    nombre completo del archivo de datos en formato Jicamarca (.r)
                         startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                         endDate            :    fecha final del rango seleccionado en formato datetime.date
                         startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                         endTime            :    tiempo final del rango seleccionado en formato datetime.time
                     Return:
                         Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                         fecha especificado, de lo contrario retorna False.
                     Excepciones:
                         Si el archivo no existe o no puede ser abierto
                         Si la cabecera no puede ser leida.
                     """
                     try:
                         fp = h5py.File(filename, 'r')
                         grp1 = fp['Data']
                     except IOError:
                         traceback.print_exc()
                         raise IOError("The file %s can't be opened" %(filename))
                     #In case has utctime attribute
                     grp2 = grp1['utctime']
                     thisUtcTime = grp2.value[0]
                     fp.close()
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime + 5*3600)
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
                     #General case
                     #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                     #-----------o----------------------------o-----------
                     #       startTime                     endTime
                     if endTime >= startTime:
                         thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
                         if numpy.any(thisUtcLog):   #If there is one block between the hours mentioned
                             return thisDatetime
                         return None
                     #If endTime < startTime then endTime belongs to the next day
                     #<<<<<<<<<<<o                            o>>>>>>>>>>>
                     #-----------o----------------------------o-----------
                     #        endTime                    startTime
                     if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
                         return None
                     if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
                         return None
                     if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
                         return None
                     return thisDatetime
                 def __setNextFileOffline(self):
                     self.fileIndex += 1
                     idFile = self.fileIndex
                     if not(idFile < len(self.filenameList)):
-                        self.dataOut.error = 'No more files'
+                        raise schainpy.admin.SchainError('No more files')
-                        return 0
                     filename = self.filenameList[idFile]
                     self.fp = h5py.File(filename, 'r')
                     self.filename = filename
                     print("Setting the file: %s"%self.filename)
                     self.__setBlockList()
                     self.__readData()
                     self.blockIndex = 0
                     return 1
                 def __setBlockList(self):
                     '''
                     Selects the data within the times defined
                     self.fp
                     self.startTime
                     self.endTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     fp = self.fp
                     startTime = self.startTime
                     endTime = self.endTime
                     grp = fp['Data']
                     thisUtcTime = grp['utctime'].value
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     return
                 def __readMetadata(self):
                     '''
                     Reads Metadata
                     '''
                     filename = self.filenameList[0]
                     fp = h5py.File(filename, 'r')
                     gp = fp['Metadata']
                     listMetaname = []
                     listMetadata = []
                     for item in list(gp.items()):
                         name = item[0]
                         if name=='variables':
                             table = gp[name][:]
                             listShapes = {}
                             for shapes in table:
                                 listShapes[shapes[0].decode()] = numpy.array([shapes[1]])
                         else:
                             data = gp[name].value
                             listMetaname.append(name)
                             listMetadata.append(data)
                     self.listShapes = listShapes
                     self.listMetaname = listMetaname
                     self.listMeta = listMetadata
                     fp.close()
                     return
                 def __readData(self):
                     grp = self.fp['Data']
                     listdataname = []
                     listdata = []
                     for item in list(grp.items()):
                         name = item[0]
                         listdataname.append(name)
                         dim = self.listShapes[name][0]
                         if dim == 0:
                             array = grp[name].value
                         else:
                             array = []
                             for i in range(dim):
                                 array.append(grp[name]['table{:02d}'.format(i)].value)
                             array = numpy.array(array)
                         listdata.append(array)
                     self.listDataname = listdataname
                     self.listData = listdata
                     return
                 def getData(self):
                     for i in range(len(self.listMeta)):
                         setattr(self.dataOut, self.listMetaname[i], self.listMeta[i])
                     for j in range(len(self.listData)):
                         dim = self.listShapes[self.listDataname[j]][0]
                         if dim == 0:
                             setattr(self.dataOut, self.listDataname[j], self.listData[j][self.blockIndex])
                         else:
                             setattr(self.dataOut, self.listDataname[j], self.listData[j][:,self.blockIndex])
                     self.dataOut.flagNoData = False
                     self.blockIndex += 1
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     if self.blockIndex == self.blocksPerFile:
                         if not(self.__setNextFileOffline()):
                             self.dataOut.flagNoData = True
                             return 0
                     self.getData()
                     return
             @MPDecorator
             class ParameterWriter(Operation):
                 '''
                 HDF5 Writer, stores parameters data in HDF5 format files
                 path:             path where the files will be stored
                 blocksPerFile:    number of blocks that will be saved in per HDF5 format file
                 mode:             selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
                 metadataList:     list of attributes that will be stored as metadata
                 dataList:         list of attributes that will be stores as data
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 metaoptchar = "M"
                 metaFile = None
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 grp = None
                 ds = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 dsList = None   #List of dictionaries with dataset properties
                 tableDim = None
                 dtype = [('name', 'S20'),('nDim', 'i')]
                 currentDay = None
                 lastTime = None
                 def __init__(self):
                     Operation.__init__(self)
                     return
                 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataList = dataList
                     self.setType = setType
                     tableList = []
                     dsList = []
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         dataAux = getattr(self.dataOut, self.dataList[i])
                         dsDict['variable'] = self.dataList[i]
                         if dataAux is None:
                             continue
                         elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
                             dsDict['nDim'] = 0
                         else:
                             dsDict['nDim'] = len(dataAux.shape)
                             dsDict['shape'] = dataAux.shape
                             dsDict['dsNumber'] = dataAux.shape[0]
                         dsList.append(dsDict)
                         tableList.append((self.dataList[i], dsDict['nDim']))
                     self.dsList = dsList
                     self.tableDim = numpy.array(tableList, dtype=self.dtype)
                     self.currentDay = self.dataOut.datatime.date()
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     timeTuple = time.localtime(currentTime)
                     dataDay = timeTuple.tm_yday
                     if self.lastTime is None:
                         self.lastTime = currentTime
                         self.currentDay = dataDay
                         return False
                     timeDiff = currentTime - self.lastTime
                     #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                     if dataDay != self.currentDay:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > 3*60*60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, setType=None):
                     self.dataOut = dataOut
                     if not(self.isConfig):
                         self.setup(path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList,
                                    setType=setType)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join(path, subfolder)
                     if os.path.exists(fullpath):
                         filesList = os.listdir(fullpath)
                         filesList = [k for k in filesList if k.startswith(self.optchar)]
                         if len( filesList ) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     self.filename = os.path.join( path, subfolder, file )
                     #Setting HDF5 File
                     self.fp = h5py.File(self.filename, 'w')
                     #write metadata
                     self.writeMetadata(self.fp)
                     #Write data
                     self.writeData(self.fp)
                 def writeMetadata(self, fp):
                     grp = fp.create_group("Metadata")
                     grp.create_dataset('variables', data=self.tableDim, dtype=self.dtype)
                     for i in range(len(self.metadataList)):
                         if not hasattr(self.dataOut, self.metadataList[i]):
                             log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
                             continue
                         value = getattr(self.dataOut, self.metadataList[i])
                         grp.create_dataset(self.metadataList[i], data=value)
                     return
                 def writeData(self, fp):
                     grp = fp.create_group("Data")
                     dtsets = []
                     data = []
                     for dsInfo in self.dsList:
                         if dsInfo['nDim'] == 0:
                             ds = grp.create_dataset(
                                 dsInfo['variable'],
                                 (self.blocksPerFile, ),
                                 chunks=True,
                                 dtype=numpy.float64)
                             dtsets.append(ds)
                             data.append((dsInfo['variable'], -1))
                         else:
                             sgrp = grp.create_group(dsInfo['variable'])
                             for i in range(dsInfo['dsNumber']):
                                 ds = sgrp.create_dataset(
                                     'table{:02d}'.format(i),
                                     (self.blocksPerFile, ) + dsInfo['shape'][1:],
                                     chunks=True)
                                 dtsets.append(ds)
                                 data.append((dsInfo['variable'], i))
                     fp.flush()
                     log.log('Creating file: {}'.format(fp.filename), self.name)
                     self.ds = dtsets
                     self.data = data
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
                         self.closeFile()
                         self.setNextFile()
                     for i, ds in enumerate(self.ds):
                         attr, ch = self.data[i]
                         if ch == -1:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)
                         else:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
                     self.fp.flush()
                     self.blockIndex += 1
                     log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
                     return
                 def closeFile(self):
                     if self.blockIndex != self.blocksPerFile:
                         for ds in self.ds:
                             ds.resize(self.blockIndex, axis=0)
                     self.fp.flush()
                     self.fp.close()
                 def close(self):
                     self.closeFile()

schainpy/model/io/jroIO_spectra.py 0 -3

             '''
             Created on Jul 2, 2014
             @author: roj-idl71
             '''
             import numpy
             from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jrodata import Spectra
             from schainpy.utils import log
             @MPDecorator
             class SpectraReader(JRODataReader, ProcessingUnit):
                 """
                 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
                 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
                                     paresCanalesIguales    * alturas * perfiles  (Self Spectra)
                                     paresCanalesDiferentes * alturas * perfiles  (Cross Spectra)
                                     canales * alturas                            (DC Channels)
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = SpectraReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         readerObj.getData()
                         print readerObj.data_spc
                         print readerObj.data_cspc
                         print readerObj.data_dc
                         if readerObj.flagNoMoreFiles:
                             break
                 """
                 pts2read_SelfSpectra = 0
                 pts2read_CrossSpectra = 0
                 pts2read_DCchannels = 0
                 ext = ".pdata"
                 optchar = "P"
                 dataOut = None
                 nRdChannels = None
                 nRdPairs = None
                 rdPairList = []
                 def __init__(self):#, **kwargs):
                     """
                     Inicializador de la clase SpectraReader para la lectura de datos de espectros.
                     Inputs:
                         dataOut    :    Objeto de la clase Spectra. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Affected:
                         self.dataOut
                     Return      : None
                     """
                     #Eliminar de la base la herencia
                     ProcessingUnit.__init__(self)#, **kwargs)
-            #         self.isConfig = False
                     self.pts2read_SelfSpectra = 0
                     self.pts2read_CrossSpectra = 0
                     self.pts2read_DCchannels = 0
                     self.datablock = None
                     self.utc = None
                     self.ext = ".pdata"
                     self.optchar = "P"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.online = 0
                     self.fp = None
                     self.idFile = None
                     self.dtype = None
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.filename = None
                     self.fileSize = None
                     self.firstHeaderSize = 0
                     self.basicHeaderSize = 24
                     self.pathList = []
                     self.lastUTTime = 0
                     self.maxTimeStep = 30
                     self.flagNoMoreFiles = 0
                     self.set = 0
                     self.path = None
                     self.delay  = 60   #seconds
                     self.nTries = 3  #quantity tries
                     self.nFiles = 3   #number of files for searching
                     self.nReadBlocks = 0
                     self.flagIsNewFile = 1
                     self.__isFirstTimeOnline = 1
-            #         self.ippSeconds = 0
                     self.flagDiscontinuousBlock = 0
                     self.flagIsNewBlock = 0
                     self.nTotalBlocks = 0
                     self.blocksize = 0
                     self.dataOut = self.createObjByDefault()
                     self.profileIndex = 1 #Always
                 def createObjByDefault(self):
                     dataObj = Spectra()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     return 1
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.nRdChannels
                         self.nRdPairs
                         self.pts2read_SelfSpectra
                         self.pts2read_CrossSpectra
                         self.pts2read_DCchannels
                         self.blocksize
                         self.dataOut.nChannels
                         self.dataOut.nPairs
                     Return:
                         None
                     """
                     self.nRdChannels = 0
                     self.nRdPairs = 0
                     self.rdPairList = []
                     for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
                         if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
                             self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
                         else:
                             self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
                             self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
                     pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
                     self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
                     self.blocksize = self.pts2read_SelfSpectra
                     if self.processingHeaderObj.flag_cspc:
                         self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
                         self.blocksize += self.pts2read_CrossSpectra
                     if self.processingHeaderObj.flag_dc:
                         self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
                         self.blocksize += self.pts2read_DCchannels
                     # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
                 def readBlock(self):
                     """
                     Lee el bloque de datos desde la posicion actual del puntero del archivo
                     (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                     (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                     es seteado a 0
                     Return: None
                     Variables afectadas:
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Exceptions:
                         Si un bloque leido no es un bloque valido
                     """
                     blockOk_flag = False
                     fpointer = self.fp.tell()
                     spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
                     spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
                         cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_dc:
                         dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
                         dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
                     if not self.processingHeaderObj.shif_fft:
                         #desplaza a la derecha en el eje 2 determinadas posiciones
                         shift = int(self.processingHeaderObj.profilesPerBlock/2)
                         spc = numpy.roll( spc, shift , axis=2 )
                         if self.processingHeaderObj.flag_cspc:
                             #desplaza a la derecha en el eje 2 determinadas posiciones
                             cspc = numpy.roll( cspc, shift, axis=2 )
                     #Dimensions : nChannels, nProfiles, nSamples
                     spc = numpy.transpose( spc, (0,2,1) )
                     self.data_spc = spc
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.transpose( cspc, (0,2,1) )
                         self.data_cspc = cspc['real'] + cspc['imag']*1j
                     else:
                         self.data_cspc = None
                     if self.processingHeaderObj.flag_dc:
                         self.data_dc = dc['real'] + dc['imag']*1j
                     else:
                         self.data_dc = None
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.getBasicHeader()
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
             #         self.dataOut.ippSeconds = self.ippSeconds
             #         self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.processingHeaderObj.profilesPerBlock
                     self.dataOut.dtype = self.dtype
             #         self.dataOut.nPairs = self.nPairs
                     self.dataOut.pairsList = self.rdPairList
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
                     xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
                     self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
                     self.dataOut.flagShiftFFT = True    #Data is always shifted
                     self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
                     self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
                 def getData(self):
                     """
                     First method to execute before "RUN" is called.
                     Copia el buffer de lectura a la clase "Spectra",
                     con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
                     Return:
 :    Si no hay mas archivos disponibles
 :    Si hizo una buena copia del buffer
                     Affected:
                         self.dataOut
                         self.flagDiscontinuousBlock
                         self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
-                        print('Process finished')
                         return 0
                     self.flagDiscontinuousBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not( self.readNextBlock() ):
                             self.dataOut.flagNoData = True
                             return 0
                     #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
                     if self.data_spc is None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.getBasicHeader()
                     self.getFirstHeader()
                     self.dataOut.data_spc = self.data_spc
                     self.dataOut.data_cspc = self.data_cspc
                     self.dataOut.data_dc = self.data_dc
                     self.dataOut.flagNoData = False
                     self.dataOut.realtime = self.online
                     return self.dataOut.data_spc
             @MPDecorator
             class SpectraWriter(JRODataWriter, Operation):
                 """
                 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 ext = ".pdata"
                 optchar = "P"
                 shape_spc_Buffer = None
                 shape_cspc_Buffer = None
                 shape_dc_Buffer = None
                 data_spc = None
                 data_cspc = None
                 data_dc = None
                 def __init__(self):
                     """
                     Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj
                     Return: None
                     """
                     Operation.__init__(self)
                     self.nTotalBlocks = 0
                     self.data_spc = None
                     self.data_cspc = None
                     self.data_dc = None
                     self.fp = None
                     self.flagIsNewFile = 1
                     self.nTotalBlocks = 0
                     self.flagIsNewBlock = 0
                     self.setFile = None
                     self.dtype = None
                     self.path = None
                     self.noMoreFiles = 0
                     self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     return 1
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shape_spc_Buffer = (self.dataOut.nChannels,
                                              self.processingHeaderObj.nHeights,
                                              self.processingHeaderObj.profilesPerBlock)
                     self.shape_cspc_Buffer = (self.dataOut.nPairs,
                                               self.processingHeaderObj.nHeights,
                                               self.processingHeaderObj.profilesPerBlock)
                     self.shape_dc_Buffer = (self.dataOut.nChannels,
                                             self.processingHeaderObj.nHeights)
                 def writeBlock(self):
                     """processingHeaderObj
                     Escribe el buffer en el file designado
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.nWriteBlocks
                     Return: None
                     """
                     spc = numpy.transpose( self.data_spc, (0,2,1) )
                     if not self.processingHeaderObj.shif_fft:
                         spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                     data = spc.reshape((-1))
                     data = data.astype(self.dtype[0])
                     data.tofile(self.fp)
                     if self.data_cspc is not None:
                         cspc = numpy.transpose( self.data_cspc, (0,2,1) )
                         data = numpy.zeros( numpy.shape(cspc), self.dtype )
                         #print 'data.shape', self.shape_cspc_Buffer
                         if not self.processingHeaderObj.shif_fft:
                             cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                         data['real'] = cspc.real
                         data['imag'] = cspc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
                     if self.data_dc is not None:
                         dc = self.data_dc
                         data = numpy.zeros( numpy.shape(dc), self.dtype )
                         data['real'] = dc.real
                         data['imag'] = dc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
             #         self.data_spc.fill(0)
             #
             #         if self.data_dc is not None:
             #             self.data_dc.fill(0)
             #
             #         if self.data_cspc is not None:
             #             self.data_cspc.fill(0)
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nWriteBlocks += 1
                     self.blockIndex += 1
             #         print "[Writing] Block = %d04" %self.blockIndex
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagDiscontinuousBlock:
                         self.data_spc.fill(0)
                         if self.dataOut.data_cspc is not None:
                             self.data_cspc.fill(0)
                         if self.dataOut.data_dc is not None:
                             self.data_dc.fill(0)
                         self.setNextFile()
                     if self.flagIsNewFile == 0:
                         self.setBasicHeader()
                     self.data_spc = self.dataOut.data_spc.copy()
                     if self.dataOut.data_cspc is not None:
                         self.data_cspc = self.dataOut.data_cspc.copy()
                     if self.dataOut.data_dc is not None:
                         self.data_dc = self.dataOut.data_dc.copy()
                     # #self.processingHeaderObj.dataBlocksPerFile)
                     if self.hasAllDataInBuffer():
             #            self.setFirstHeader()
                         self.writeNextBlock()
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
                     '''
                     dtype_width = self.getDtypeWidth()
                     pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
                     pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
                     blocksize = (pts2write_SelfSpectra*dtype_width)
                     if self.dataOut.data_cspc is not None:
                         pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
                         blocksize += (pts2write_CrossSpectra*dtype_width*2)
                     if self.dataOut.data_dc is not None:
                         pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
                         blocksize += (pts2write_DCchannels*dtype_width*2)
             #         blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.processingHeaderObj.dtype = 1 # Spectra
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
                     self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
                     self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
                     self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
                     if self.processingHeaderObj.totalSpectra > 0:
                         channelList = []
                         for channel in range(self.dataOut.nChannels):
                             channelList.append(channel)
                             channelList.append(channel)
                         pairsList = []
                         if self.dataOut.nPairs > 0:
                             for pair in self.dataOut.pairsList:
                                 pairsList.append(pair[0])
                                 pairsList.append(pair[1])
                         spectraComb = channelList + pairsList
                         spectraComb = numpy.array(spectraComb, dtype="u1")
                         self.processingHeaderObj.spectraComb = spectraComb
                     if self.dataOut.code is not None:
                         self.processingHeaderObj.code = self.dataOut.code
                         self.processingHeaderObj.nCode = self.dataOut.nCode
                         self.processingHeaderObj.nBaud = self.dataOut.nBaud
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                     self.processingHeaderObj.processFlags = self.getProcessFlags()
                     self.setBasicHeader()

schainpy/model/proc/jroproc_base.py +28 -53

             '''
             Updated for multiprocessing
             Author : Sergio Cortez
             Jan 2018
             Abstract:
                 Base class for processing units and operations. A decorator provides multiprocessing features and interconnect the processes created.
                 The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
                 The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
             Based on:
                 $Author: murco $
                 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
             '''
             import os
             import inspect
             import zmq
             import time
             import pickle
+            import traceback
             from queue import Queue
             from threading import Thread
             from multiprocessing import Process
-            from zmq.utils.monitor import recv_monitor_message
             from schainpy.utils import log
             class ProcessingUnit(object):
                 """
                 Update - Jan 2018 - MULTIPROCESSING
                 All the "call" methods present in the previous base were removed.
                 The majority of operations are independant processes, thus
                 the decorator is in charge of communicate the operation processes
                 with the proccessing unit via IPC.
                 The constructor does not receive any argument. The remaining methods
                 are related with the operations to execute.
                 """
                 def __init__(self):
                     self.dataIn = None
                     self.dataOut = None
                     self.isConfig = False
                     self.operations = []
                     self.plots = []
                 def getAllowedArgs(self):
                     if hasattr(self, '__attrs__'):
                         return self.__attrs__
                     else:
                         return inspect.getargspec(self.run).args
                 def addOperation(self, conf, operation):
                     """
                     This method is used in the controller, and update the dictionary containing the operations to execute. The dict
                     posses the id of the operation process (IPC purposes)
                         Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
                         identificador asociado a este objeto.
                         Input:
                             object    :    objeto de la clase "Operation"
                         Return:
                             objId    :    identificador del objeto, necesario para comunicar con master(procUnit)
                     """
                     self.operations.append(
                         (operation, conf.type, conf.id, conf.getKwargs()))
                     if 'plot' in self.name.lower():
                         self.plots.append(operation.CODE)
                 def getOperationObj(self, objId):
                     if objId not in list(self.operations.keys()):
                         return None
                     return self.operations[objId]
                 def operation(self, **kwargs):
                     """
                     Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
                     atributos del objeto dataOut
                     Input:
                         **kwargs    :    Diccionario de argumentos de la funcion a ejecutar
                     """
                     raise NotImplementedError
                 def setup(self):
                     raise NotImplementedError
                 def run(self):
                     raise NotImplementedError
                 def close(self):
                     return
             class Operation(object):
                 """
                 Update - Jan 2018 - MULTIPROCESSING
                 Most of the methods remained the same. The decorator parse the arguments and executed the run() method for each process.
                 The constructor doe snot receive any argument, neither the baseclass.
                     Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
                     y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
                     acumulacion dentro de esta clase
                     Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
                 """
                 def __init__(self):
                     self.id = None
                     self.isConfig = False
                     if not hasattr(self, 'name'):
                         self.name = self.__class__.__name__
                 def getAllowedArgs(self):
                     if hasattr(self, '__attrs__'):
                         return self.__attrs__
                     else:
                         return inspect.getargspec(self.run).args
                 def setup(self):
                     self.isConfig = True
                     raise NotImplementedError
                 def run(self, dataIn, **kwargs):
                     """
                     Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
                     atributos del objeto dataIn.
                     Input:
                         dataIn    :    objeto del tipo JROData
                     Return:
                         None
                     Affected:
                         __buffer    :    buffer de recepcion de datos.
                     """
                     if not self.isConfig:
                         self.setup(**kwargs)
                     raise NotImplementedError
                 def close(self):
                     return
             class InputQueue(Thread):
                 '''
                 Class to hold input data for Proccessing Units and external Operations,
                 '''
                 def __init__(self, project_id, inputId):
                     Thread.__init__(self)
                     self.queue = Queue()
                     self.project_id = project_id
                     self.inputId = inputId
                 def run(self):
                     c = zmq.Context()
                     self.receiver = c.socket(zmq.SUB)
                     self.receiver.connect(
                         'ipc:///tmp/schain/{}_pub'.format(self.project_id))
                     self.receiver.setsockopt(zmq.SUBSCRIBE, self.inputId.encode())
                     while True:
                         self.queue.put(self.receiver.recv_multipart()[1])
                 def get(self):
                     return pickle.loads(self.queue.get())
             def MPDecorator(BaseClass):
                 """
                 Multiprocessing class decorator
                 This function add multiprocessing features to a BaseClass. Also, it handle
                 the communication beetween processes (readers, procUnits and operations).
                 """
                 class MPClass(BaseClass, Process):
                     def __init__(self, *args, **kwargs):
                         super(MPClass, self).__init__()
                         Process.__init__(self)
                         self.operationKwargs = {}
                         self.args = args
                         self.kwargs = kwargs
                         self.sender = None
                         self.receiver = None
                         self.i = 0
                         self.name = BaseClass.__name__
                         if 'plot' in self.name.lower() and not self.name.endswith('_'):
                             self.name = '{}{}'.format(self.CODE.upper(), 'Plot')
-                        self.start_time = time.time()
-                        if len(self.args) is 3:
+                        self.start_time = time.time()
-                            self.typeProc = "ProcUnit"
                         self.id = args[0]
                         self.inputId = args[1]
                         self.project_id = args[2]
-                        elif len(self.args) is 2:
+                        self.err_queue = args[3]
-                            self.id = args[0]
+                        self.typeProc = args[4]
-                            self.inputId = args[0]
-                            self.project_id = args[1]
-                            self.typeProc = "Operation"
                         self.queue = InputQueue(self.project_id, self.inputId)
+                        self.err_queue.put('#_start_#')
                     def subscribe(self):
                         '''
                         This function start the input queue.
                         '''
                         self.queue.start()
                     def listen(self):
                         '''
                         This function waits for objects
                         '''
                         return self.queue.get()
                     def set_publisher(self):
                         '''
                         This function create a socket for publishing purposes.
                         '''
                         time.sleep(1)
                         c = zmq.Context()
                         self.sender = c.socket(zmq.PUB)
                         self.sender.connect(
                             'ipc:///tmp/schain/{}_sub'.format(self.project_id))
                     def publish(self, data, id):
                         '''
                         This function publish an object, to a specific topic.
                         For Read Units (inputId == None) adds a little delay
                         to avoid data loss
                         '''
                         if self.inputId is None:
                             self.i += 1
-                            if self.i % 100 == 0:
+                            if self.i % 80 == 0:
                                 self.i = 0
                                 time.sleep(0.01)
                         self.sender.send_multipart([str(id).encode(), pickle.dumps(data)])
                     def runReader(self):
                         '''
                         Run fuction for read units
                         '''
                         while True:
+                            try:
                                 BaseClass.run(self, **self.kwargs)
+                            except:
+                                err = traceback.format_exc()
+                                if 'No more files' in err:
+                                    log.warning('No more files to read', self.name)
+                                else:
+                                    self.err_queue.put('{}|{}'.format(self.name, err))
+                                self.dataOut.error = True
                             for op, optype, opId, kwargs in self.operations:
                                 if optype == 'self' and not self.dataOut.flagNoData:
                                     op(**kwargs)
                                 elif optype == 'other' and not self.dataOut.flagNoData:
                                     self.dataOut = op.run(self.dataOut, **self.kwargs)
                                 elif optype == 'external':
                                     self.publish(self.dataOut, opId)
                             if self.dataOut.flagNoData and not self.dataOut.error:
                                 continue
                             self.publish(self.dataOut, self.id)
                             if self.dataOut.error:
-                                log.error(self.dataOut.error, self.name)
-                                # self.sender.send_multipart([str(self.project_id).encode(), 'end'.encode()])
                                 break
-                        time.sleep(1)
+                        time.sleep(0.5)
                     def runProc(self):
                         '''
                         Run function for proccessing units
                         '''
                         while True:
                             self.dataIn = self.listen()
                             if self.dataIn.flagNoData and self.dataIn.error is None:
                                 continue
+                            elif not self.dataIn.error:
+                                try:
                                     BaseClass.run(self, **self.kwargs)
+                                except:
-                            if self.dataIn.error:
+                                    self.err_queue.put('{}|{}'.format(self.name, traceback.format_exc()))
+                                    self.dataOut.error = True
+                            elif self.dataIn.error:
                                 self.dataOut.error = self.dataIn.error
                                 self.dataOut.flagNoData = True
                             for op, optype, opId, kwargs in self.operations:
                                 if optype == 'self' and not self.dataOut.flagNoData:
                                     op(**kwargs)
                                 elif optype == 'other' and not self.dataOut.flagNoData:
                                     self.dataOut = op.run(self.dataOut, **kwargs)
                                 elif optype == 'external' and not self.dataOut.flagNoData:
                                     self.publish(self.dataOut, opId)
-                            if not self.dataOut.flagNoData or self.dataOut.error:
                             self.publish(self.dataOut, self.id)
                             for op, optype, opId, kwargs in self.operations:
-                                    if optype == 'self' and self.dataOut.error:
+                                if optype == 'external' and self.dataOut.error:
-                                        op(**kwargs)
-                                    elif optype == 'other' and self.dataOut.error:
-                                        self.dataOut = op.run(self.dataOut, **kwargs)
-                                    elif optype == 'external' and self.dataOut.error:
                                     self.publish(self.dataOut, opId)
-                            if self.dataIn.error:
+                            if self.dataOut.error:
                                 break
-                        time.sleep(1)
+                        time.sleep(0.5)
                     def runOp(self):
                         '''
                         Run function for external operations (this operations just receive data
                         ex: plots, writers, publishers)
                         '''
                         while True:
                             dataOut = self.listen()
+                            if not dataOut.error:
                                 BaseClass.run(self, dataOut, **self.kwargs)
+                            else:
-                            if dataOut.error:
                                 break
-                        time.sleep(1)
                     def run(self):
                         if self.typeProc is "ProcUnit":
                             if self.inputId is not None:
                                 self.subscribe()
                             self.set_publisher()
                             if 'Reader' not in BaseClass.__name__:
                                 self.runProc()
                             else:
                                 self.runReader()
                         elif self.typeProc is "Operation":
                             self.subscribe()
                             self.runOp()
                         else:
                             raise ValueError("Unknown type")
                         self.close()
-                    def event_monitor(self, monitor):
-                        events = {}
-                        for name in dir(zmq):
-                            if name.startswith('EVENT_'):
-                                value = getattr(zmq, name)
-                                events[value] = name
-                        while monitor.poll():
-                            evt = recv_monitor_message(monitor)
-                            if evt['event'] == 32:
-                                self.connections += 1
-                            if evt['event'] == 512:
-                                pass
-                            evt.update({'description': events[evt['event']]})
-                            if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
-                                break
-                        monitor.close()
-                        print('event monitor thread done!')
                     def close(self):
                         BaseClass.close(self)
+                        self.err_queue.put('#_end_#')
                         if self.sender:
                             self.sender.close()
                         if self.receiver:
                             self.receiver.close()
                         log.success('Done...(Time:{:4.2f} secs)'.format(time.time()-self.start_time), self.name)
                 return MPClass

schainpy/model/proc/jroproc_parameters.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages