schain Commit - r1287:af11e4aac00c · Jicamarca Repository

Rewrite controller, remove MPDecorator to units (keep for plots an writers) use of queues for interproc comm instead of zmq, self operations are no longer supported

Juan C. Espinoza -

r1287:af11e4aac00c

parent child

Context file:

r1287:af11e4aac00c

Collapse all files

The requested changes are too big and content was truncated. Show full diff

schainpy/cli/cli.py +7 -10

             import click
-            import schainpy
             import subprocess
             import os
             import sys
             import glob
-            from multiprocessing import cpu_count
+            import schainpy
             from schainpy.controller import Project
             from schainpy.model import Operation, ProcessingUnit
             from schainpy.utils import log
             from importlib import import_module
             from pydoc import locate
             from fuzzywuzzy import process
             from schainpy.cli import templates
             import inspect
             try:
                 from queue import Queue
             except:
                 from Queue import Queue
             def getProcs():
                 modules = dir(schainpy.model)
                 procs = check_module(modules, 'processing')
                 try:
                     procs.remove('ProcessingUnit')
                 except Exception as e:
                     pass
                 return procs
             def getOperations():
                 module = dir(schainpy.model)
                 noProcs = [x for x in module if not x.endswith('Proc')]
                 operations = check_module(noProcs, 'operation')
                 try:
                     operations.remove('Operation')
                     operations.remove('Figure')
                     operations.remove('Plot')
                 except Exception as e:
                     pass
                 return operations
             def getArgs(op):
                 module = locate('schainpy.model.{}'.format(op))
                 try:
-                    obj = module(1,2,3,Queue(),5,6)
+                    obj = module(1, 2, 3, Queue())
                 except:
                     obj = module()
                 if hasattr(obj, '__attrs__'):
                     args = obj.__attrs__
                 else:
                     if hasattr(obj, 'myrun'):
                         args = inspect.getfullargspec(obj.myrun).args
                     else:
                         args = inspect.getfullargspec(obj.run).args
                 try:
                     args.remove('self')
                 except Exception as e:
                     pass
                 try:
                     args.remove('dataOut')
                 except Exception as e:
                     pass
                 return args
             def getDoc(obj):
                 module = locate('schainpy.model.{}'.format(obj))
                 try:
-                    obj = module(1,2,3,Queue(),5,6)
+                    obj = module(1, 2, 3, Queue())
                 except:
                     obj = module()
                 return obj.__doc__
             def getAll():
                 modules = getOperations()
                 modules.extend(getProcs())
                 return modules
             def print_version(ctx, param, value):
                 if not value or ctx.resilient_parsing:
                     return
                 click.echo(schainpy.__version__)
                 ctx.exit()
             PREFIX = 'experiment'
             @click.command()
             @click.option('--version', '-v', is_flag=True, callback=print_version, help='SChain version', type=str)
             @click.argument('command', default='run', required=True)
             @click.argument('nextcommand', default=None, required=False, type=str)
             def main(command, nextcommand, version):
                 """COMMAND LINE INTERFACE FOR SIGNAL CHAIN - JICAMARCA RADIO OBSERVATORY V3.0\n
-                    Available commands.\n
+                    Available commands:\n
                     xml: runs a schain XML generated file\n
-                    run: runs any python script starting 'experiment_'\n
+                    run: runs any python script'\n
                     generate: generates a template schain script\n
                     list: return a list of available procs and operations\n
                     search: return avilable operations, procs or arguments of the given
                             operation/proc\n"""
                 if command == 'xml':
                     runFromXML(nextcommand)
                 elif command == 'generate':
                     generate()
                 elif command == 'test':
                     test()
                 elif command == 'run':
                     runschain(nextcommand)
                 elif command == 'search':
                     search(nextcommand)
                 elif command == 'list':
                     cmdlist(nextcommand)
                 else:
                     log.error('Command {} is not defined'.format(command))
             def check_module(possible, instance):
                 def check(x):
                     try:
                         instancia = locate('schainpy.model.{}'.format(x))
                         ret = instancia.proc_type == instance
                         return ret
                     except Exception as e:
                         return False
                 clean = clean_modules(possible)
                 return [x for x in clean if check(x)]
             def clean_modules(module):
                 noEndsUnder = [x for x in module if not x.endswith('__')]
                 noStartUnder = [x for x in noEndsUnder if not x.startswith('__')]
                 noFullUpper = [x for x in noStartUnder if not x.isupper()]
                 return noFullUpper
             def cmdlist(nextcommand):
                 if nextcommand is None:
                     log.error('Missing argument, available arguments: procs, operations', '')
                 elif nextcommand == 'procs':
                     procs = getProcs()
                     log.success(
                         'Current ProcessingUnits are:\n  {}'.format('\n  '.join(procs)), '')
                 elif nextcommand == 'operations':
                     operations = getOperations()
                     log.success('Current Operations are:\n  {}'.format(
                         '\n  '.join(operations)), '')
                 else:
                     log.error('Wrong argument', '')
             def search(nextcommand):
                 if nextcommand is None:
                     log.error('There is no Operation/ProcessingUnit to search', '')
                 else:
                     try:
                         args = getArgs(nextcommand)
                         doc = getDoc(nextcommand)
-                        if len(args) == 0:
-                            log.success('\n{} has no arguments'.format(nextcommand), '')
-                        else:
                         log.success('{}\n{}\n\narguments:\n  {}'.format(
-                                nextcommand, doc, ', '.join(args)), '')
+                            nextcommand, doc, ', '.join(args)), ''
+                            )
                     except Exception as e:
                         log.error('Module `{}` does not exists'.format(nextcommand), '')
                         allModules = getAll()
                         similar = [t[0] for t in process.extract(nextcommand, allModules, limit=12) if t[1]>80]
                         log.success('Possible modules are: {}'.format(', '.join(similar)), '')
             def runschain(nextcommand):
                 if nextcommand is None:
                     currentfiles = glob.glob('./{}_*.py'.format(PREFIX))
                     numberfiles = len(currentfiles)
                     if numberfiles > 1:
                         log.error('There is more than one file to run')
                     elif numberfiles == 1:
                         subprocess.call(['python ' + currentfiles[0]], shell=True)
                     else:
                         log.error('There is no file to run')
                 else:
                     try:
                         subprocess.call(['python ' + nextcommand], shell=True)
                     except Exception as e:
                         log.error("I cannot run the file. Does it exists?")
             def basicInputs():
                 inputs = {}
                 inputs['name'] = click.prompt(
                     'Name of the project', default="project", type=str)
                 inputs['desc'] = click.prompt(
                     'Enter a description', default="A schain project", type=str)
                 inputs['multiprocess'] = click.prompt(
                     '''Select data type:
                 - Voltage (*.r):                [1]
                 - Spectra (*.pdata):            [2]
                 - Voltage and Spectra (*.r):    [3]
                 -->''', type=int)
                 inputs['path'] = click.prompt('Data path', default=os.getcwd(
                 ), type=click.Path(exists=True, resolve_path=True))
                 inputs['startDate'] = click.prompt(
                     'Start date', default='1970/01/01', type=str)
                 inputs['endDate'] = click.prompt(
                     'End date', default='2018/12/31', type=str)
                 inputs['startHour'] = click.prompt(
                     'Start hour', default='00:00:00', type=str)
                 inputs['endHour'] = click.prompt('End hour', default='23:59:59', type=str)
                 inputs['figpath'] = inputs['path'] + '/figs'
                 return inputs
             def generate():
                 inputs = basicInputs()
                 if inputs['multiprocess'] == 1:
                     current = templates.voltage.format(**inputs)
                 elif inputs['multiprocess'] == 2:
                     current = templates.spectra.format(**inputs)
                 elif inputs['multiprocess'] == 3:
                     current = templates.voltagespectra.format(**inputs)
                 scriptname = '{}_{}.py'.format(PREFIX, inputs['name'])
                 script = open(scriptname, 'w')
                 try:
                     script.write(current)
                     log.success('Script {} generated'.format(scriptname))
                 except Exception as e:
                     log.error('I cannot create the file. Do you have writing permissions?')
             def test():
                 log.warning('testing')
             def runFromXML(filename):
                 controller = Project()
                 if not controller.readXml(filename):
                     return
                 controller.start()
                 return

schainpy/controller.py +258 -900

This diff has been collapsed as it changes many lines, (1158 lines changed) Show them Hide them
	@@ -1,1290 +1,648
	1	'''	1	'''
	2	Updated on January , 2018, for multiprocessing purposes	2	Main routines to create a Signal Chain project
	3	Author: Sergio Cortez
	4	Created on September , 2012
	5	'''	3	'''
	6	from platform import python_version	4
			5	import re
	7	import sys	6	import sys
	8	import ast	7	import ast
	9	import datetime	8	import datetime
	10	import traceback	9	import traceback
	11	import math
	12	import time	10	import time
	13	import zmq	11	from multiprocessing import Process, Queue
	14	from multiprocessing import Process, Queue, Event, Value, cpu_count
	15	from threading import Thread	12	from threading import Thread
	16	from xml.etree.ElementTree import ElementTree, Element, SubElement, ~~tostring~~	13	from xml.etree.ElementTree import ElementTree, Element, SubElement
	17	from xml.dom import minidom
	18
	19		14
	20	from schainpy.admin import Alarm, SchainWarning	15	from schainpy.admin import Alarm, SchainWarning
	21	from schainpy.model import *	16	from schainpy.model import *
	22	from schainpy.utils import log	17	from schainpy.utils import log
	23		18
	24		19
	25	DTYPES = {	20	class ConfBase():
	26	'Voltage': '.r',
	27	'Spectra': '.pdata'
	28	}
	29
	30
	31	def MPProject(project, n=cpu_count()):
	32	'''
	33	Project wrapper to run schain in n processes
	34	'''
	35
	36	rconf = project.getReadUnitObj()
	37	op = rconf.getOperationObj('run')
	38	dt1 = op.getParameterValue('startDate')
	39	dt2 = op.getParameterValue('endDate')
	40	tm1 = op.getParameterValue('startTime')
	41	tm2 = op.getParameterValue('endTime')
	42	days = (dt2 - dt1).days
	43
	44	for day in range(days + 1):
	45	skip = 0
	46	cursor = 0
	47	processes = []
	48	dt = dt1 + datetime.timedelta(day)
	49	dt_str = dt.strftime('%Y/%m/%d')
	50	reader = JRODataReader()
	51	paths, files = reader.searchFilesOffLine(path=rconf.path,
	52	startDate=dt,
	53	endDate=dt,
	54	startTime=tm1,
	55	endTime=tm2,
	56	ext=DTYPES[rconf.datatype])
	57	nFiles = len(files)
	58	if nFiles == 0:
	59	continue
	60	skip = int(math.ceil(nFiles / n))
	61	while nFiles > cursor * skip:
	62	rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
	63	skip=skip)
	64	p = project.clone()
	65	p.start()
	66	processes.append(p)
	67	cursor += 1
	68
	69	def beforeExit(exctype, value, trace):
	70	for process in processes:
	71	process.terminate()
	72	process.join()
	73	print(traceback.print_tb(trace))
	74
	75	sys.excepthook = beforeExit
	76
	77	for process in processes:
	78	process.join()
	79	process.terminate()
	80
	81	time.sleep(3)
	82
	83	def wait(context):
	84
	85	time.sleep(1)
	86	c = zmq.Context()
	87	receiver = c.socket(zmq.SUB)
	88	receiver.connect('ipc:///tmp/schain_{}_pub'.format(self.id))
	89	receiver.setsockopt(zmq.SUBSCRIBE, self.id.encode())
	90	msg = receiver.recv_multipart()[1]
	91	context.terminate()
	92
	93	class ParameterConf():
	94
	95	id = None
	96	name = None
	97	value = None
	98	format = None
	99
	100	__formated_value = None
	101
	102	ELEMENTNAME = 'Parameter'
	103
	104	def __init__(self):
	105
	106	self.format = 'str'
	107
	108	def getElementName(self):
	109
	110	return self.ELEMENTNAME
	111
	112	def getValue(self):
	113
	114	value = self.value
	115	format = self.format
	116
	117	if self.__formated_value != None:
	118
	119	return self.__formated_value
	120
	121	if format == 'obj':
	122	return value
	123
	124	if format == 'str':
	125	self.__formated_value = str(value)
	126	return self.__formated_value
	127
	128	if value == '':
	129	raise ValueError('%s: This parameter value is empty' % self.name)
	130
	131	if format == 'list':
	132	strList = [s.strip() for s in value.split(',')]
	133	self.__formated_value = strList
	134
	135	return self.__formated_value
	136
	137	if format == 'intlist':
	138	'''
	139	Example:
	140	value = (0,1,2)
	141	'''
	142
	143	new_value = ast.literal_eval(value)
	144
	145	if type(new_value) not in (tuple, list):
	146	new_value = [int(new_value)]
	147
	148	self.__formated_value = new_value
	149
	150	return self.__formated_value
	151
	152	if format == 'floatlist':
	153	'''
	154	Example:
	155	value = (0.5, 1.4, 2.7)
	156	'''
	157
	158	new_value = ast.literal_eval(value)
	159
	160	if type(new_value) not in (tuple, list):
	161	new_value = [float(new_value)]
	162
	163	self.__formated_value = new_value
	164
	165	return self.__formated_value
	166
	167	if format == 'date':
	168	strList = value.split('/')
	169	intList = [int(x) for x in strList]
	170	date = datetime.date(intList[0], intList[1], intList[2])
	171
	172	self.__formated_value = date
	173
	174	return self.__formated_value
	175
	176	if format == 'time':
	177	strList = value.split(':')
	178	intList = [int(x) for x in strList]
	179	time = datetime.time(intList[0], intList[1], intList[2])
	180
	181	self.__formated_value = time
	182
	183	return self.__formated_value
	184
	185	if format == 'pairslist':
	186	'''
	187	Example:
	188	value = (0,1),(1,2)
	189	'''
	190
	191	new_value = ast.literal_eval(value)
	192
	193	if type(new_value) not in (tuple, list):
	194	raise ValueError('%s has to be a tuple or list of pairs' % value)
	195
	196	if type(new_value[0]) not in (tuple, list):
	197	if len(new_value) != 2:
	198	raise ValueError('%s has to be a tuple or list of pairs' % value)
	199	new_value = [new_value]
	200
	201	for thisPair in new_value:
	202	if len(thisPair) != 2:
	203	raise ValueError('%s has to be a tuple or list of pairs' % value)
	204
	205	self.__formated_value = new_value
	206
	207	return self.__formated_value
	208
	209	if format == 'multilist':
	210	'''
	211	Example:
	212	value = (0,1,2),(3,4,5)
	213	'''
	214	multiList = ast.literal_eval(value)
	215
	216	if type(multiList[0]) == int:
	217	multiList = ast.literal_eval('(' + value + ')')
	218
	219	self.__formated_value = multiList
	220
	221	return self.__formated_value
	222
	223	if format == 'bool':
	224	value = int(value)
	225
	226	if format == 'int':
	227	value = float(value)
	228
	229	format_func = eval(format)
	230
	231	self.__formated_value = format_func(value)
	232
	233	return self.__formated_value
	234
	235	def updateId(self, new_id):
	236
	237	self.id = str(new_id)
	238
	239	def setup(self, id, name, value, format='str'):
	240	self.id = str(id)
	241	self.name = name
	242	if format == 'obj':
	243	self.value = value
	244	else:
	245	self.value = str(value)
	246	self.format = str.lower(format)
	247
	248	self.getValue()
	249
	250	return 1
	251
	252	def update(self, name, value, format='str'):
	253
	254	self.name = name
	255	self.value = str(value)
	256	self.format = format
	257
	258	def makeXml(self, opElement):
	259	if self.name not in ('queue',):
	260	parmElement = SubElement(opElement, self.ELEMENTNAME)
	261	parmElement.set('id', str(self.id))
	262	parmElement.set('name', self.name)
	263	parmElement.set('value', self.value)
	264	parmElement.set('format', self.format)
	265
	266	def readXml(self, parmElement):
	267
	268	self.id = parmElement.get('id')
	269	self.name = parmElement.get('name')
	270	self.value = parmElement.get('value')
	271	self.format = str.lower(parmElement.get('format'))
	272
	273	# Compatible with old signal chain version
	274	if self.format == 'int' and self.name == 'idfigure':
	275	self.name = 'id'
	276
	277	def printattr(self):
	278
	279	print('Parameter[%s]: name = %s, value = %s, format = %s, project_id = %s' % (self.id, self.name, self.value, self.format, self.project_id))
	280
	281	class OperationConf():
	282
	283	ELEMENTNAME = 'Operation'
	284		21
	285	def __init__(self):	22	def __init__(self):
	286		23
	287	self.id = '0'	24	self.id = '0'
	288	self.name = None	25	self.name = None
	289	self.priority = None	26	self.priority = None
	290	self.~~topic~~ = ~~None~~	27	self.parameters = {}
	291		28	self.object = None
	292	def __getNewId(self):	29	self.operations = []
	293
	294	return int(self.id) * 10 + len(self.parmConfObjList) + 1
	295		30
	296	def getId(self):	31	def getId(self):
			32
	297	return self.id	33	return self.id
	298		34
			35	def getNewId(self):
			36
			37	return int(self.id) * 10 + len(self.operations) + 1
			38
	299	def updateId(self, new_id):	39	def updateId(self, new_id):
	300		40
	301	self.id = str(new_id)	41	self.id = str(new_id)
	302		42
	303	n = 1	43	n = 1
	304	for ~~parmObj~~ in self.~~parmConfObjList~~:	44	for conf in self.operations:
	305		45	conf_id = str(int(new_id) * 10 + n)
	306	idParm = str(int(new_id) * 10 + n)	46	conf.updateId(conf_id)
	307	parmObj.updateId(idParm)
	308
	309	n += 1	47	n += 1
	310		48
	311	def get~~ElementName~~(self):	49	def getKwargs(self):
	312
	313	return self.ELEMENTNAME
	314
	315	def getParameterObjList(self):
	316		50
	317	return self.parmConfObjList	51	params = {}
	318		52
	319	def getParameterObj(self, parameterName):	53	for key, value in self.parameters.items():
			54	if value not in (None, '', ' '):
			55	params[key] = value
	320		56
	321	for parmConfObj in self.parmConfObjList:	57	return params
	322		58
	323	if parmConfObj.name != parameterName:	59	def update(self, **kwargs):
	324	continue
	325		60
	326	return parmConfObj	61	for key, value in kwargs.items():
			62	self.addParameter(name=key, value=value)
	327		63
	328	return None	64	def addParameter(self, name, value, format=None):
			65	'''
			66	'''
	329		67
	330	def getParameterObjfromValue(self, parameterValue):	68	if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
			69	self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
			70	elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
			71	self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
			72	else:
			73	try:
			74	self.parameters[name] = ast.literal_eval(value)
			75	except:
			76	if isinstance(value, str) and ',' in value:
			77	self.parameters[name] = value.split(',')
			78	else:
			79	self.parameters[name] = value
	331		80
	332	for parmConfObj in self.parmConfObjList:	81	def getParameters(self):
	333		82
	334	if parmConfObj.getValue() != parameterValue:	83	params = {}
	335	continue	84	for key, value in self.parameters.items():
			85	s = type(value).__name__
			86	if s == 'date':
			87	params[key] = value.strftime('%Y/%m/%d')
			88	elif s == 'time':
			89	params[key] = value.strftime('%H:%M:%S')
			90	else:
			91	params[key] = str(value)
	336		92
	337	return par~~mConfObj~~.~~getValue~~()	93	return params
	338		94
	339	return None	95	def makeXml(self, element):
	340		96
	341	def getParameterValue(self, parameterName):	97	xml = SubElement(element, self.ELEMENTNAME)
			98	for label in self.xml_labels:
			99	xml.set(label, str(getattr(self, label)))
	342		100
	343	~~parameterObj~~ = self.getParameter~~Obj~~(~~parameterName~~)	101	for key, value in self.getParameters().items():
			102	xml_param = SubElement(xml, 'Parameter')
			103	xml_param.set('name', key)
			104	xml_param.set('value', value)
	344		105
	345	# if not parameterObj:	106	for conf in self.operations:
	346	# return None	107	conf.makeXml(xml)
	347		108
	348	value = parameterObj.getValue()	109	def __str__(self):
	349		110
	350	return value	111	if self.ELEMENTNAME == 'Operation':
			112	s = ' {}[id={}]\n'.format(self.name, self.id)
			113	else:
			114	s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
	351		115
	352	def getKwargs(self):	116	for key, value in self.parameters.items():
			117	if self.ELEMENTNAME == 'Operation':
			118	s += ' {}: {}\n'.format(key, value)
			119	else:
			120	s += ' {}: {}\n'.format(key, value)
	353		121
	354	kwargs = {}	122	for conf in self.operations:
			123	s += str(conf)
	355		124
	356	for parmConfObj in self.parmConfObjList:	125	return s
	357	if self.name == 'run' and parmConfObj.name == 'datatype':
	358	continue
	359		126
	360	kwargs[parmConfObj.name] = parmConfObj.getValue()	127	class OperationConf(ConfBase):
	361		128
	362	return kwargs	129	ELEMENTNAME = 'Operation'
			130	xml_labels = ['id', 'name']
	363		131
	364	def setup(self, id, name, priority, ~~type~~, project_id, err_queue, ~~lock~~):	132	def setup(self, id, name, priority, project_id, err_queue):
	365		133
	366	self.id = str(id)	134	self.id = str(id)
	367	self.project_id = project_id	135	self.project_id = project_id
	368	self.name = name	136	self.name = name
	369	self.type = ~~type~~	137	self.type = 'other'
	370	self.priority = priority
	371	self.err_queue = err_queue	138	self.err_queue = err_queue
	372	self.lock = lock
	373	self.parmConfObjList = []
	374
	375	def removeParameters(self):
	376
	377	for obj in self.parmConfObjList:
	378	del obj
	379
	380	self.parmConfObjList = []
	381		139
	382	def addParameter(self, name, value, format='str'):	140	def readXml(self, element, project_id, err_queue):
	383		141
	384	if value is None:	142	self.id = element.get('id')
	385	return None	143	self.name = element.get('name')
	386	id = self.__getNewId()	144	self.type = 'other'
	387
	388	parmConfObj = ParameterConf()
	389	if not parmConfObj.setup(id, name, value, format):
	390	return None
	391
	392	self.parmConfObjList.append(parmConfObj)
	393
	394	return parmConfObj
	395
	396	def changeParameter(self, name, value, format='str'):
	397
	398	parmConfObj = self.getParameterObj(name)
	399	parmConfObj.update(name, value, format)
	400
	401	return parmConfObj
	402
	403	def makeXml(self, procUnitElement):
	404
	405	opElement = SubElement(procUnitElement, self.ELEMENTNAME)
	406	opElement.set('id', str(self.id))
	407	opElement.set('name', self.name)
	408	opElement.set('type', self.type)
	409	opElement.set('priority', str(self.priority))
	410
	411	for parmConfObj in self.parmConfObjList:
	412	parmConfObj.makeXml(opElement)
	413
	414	def readXml(self, opElement, project_id):
	415
	416	self.id = opElement.get('id')
	417	self.name = opElement.get('name')
	418	self.type = opElement.get('type')
	419	self.priority = opElement.get('priority')
	420	self.project_id = str(project_id)	145	self.project_id = str(project_id)
			146	self.err_queue = err_queue
	421		147
	422	# Compatible with old signal chain version	148	for elm in element.iter('Parameter'):
	423	# Use of 'run' method instead 'init'	149	self.addParameter(elm.get('name'), elm.get('value'))
	424	if self.type == 'self' and self.name == 'init':
	425	self.name = 'run'
	426
	427	self.parmConfObjList = []
	428
	429	parmElementList = opElement.iter(ParameterConf().getElementName())
	430
	431	for parmElement in parmElementList:
	432	parmConfObj = ParameterConf()
	433	parmConfObj.readXml(parmElement)
	434
	435	# Compatible with old signal chain version
	436	# If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
	437	if self.type != 'self' and self.name == 'Plot':
	438	if parmConfObj.format == 'str' and parmConfObj.name == 'type':
	439	self.name = parmConfObj.value
	440	continue
	441
	442	self.parmConfObjList.append(parmConfObj)
	443
	444	def printattr(self):
	445
	446	print('%s[%s]: name = %s, type = %s, priority = %s, project_id = %s' % (self.ELEMENTNAME,
	447	self.id,
	448	self.name,
	449	self.type,
	450	self.priority,
	451	self.project_id))
	452
	453	for parmConfObj in self.parmConfObjList:
	454	parmConfObj.printattr()
	455		150
	456	def createObject(self):	151	def createObject(self):
	457		152
	458	className = eval(self.name)	153	className = eval(self.name)
	459		154
	460	if self.type == 'other':	155	if 'Plot' in self.name or 'Writer' in self.name:
	461	opObj = className()
	462	elif self.type == 'external':
	463	kwargs = self.getKwargs()	156	kwargs = self.getKwargs()
	464	opObj = className(self.id, self.id, self.project_id, self.err_queue, ~~self~~.~~lock~~, ~~'Operation'~~, **kwargs)	157	opObj = className(self.id, self.id, self.project_id, self.err_queue, **kwargs)
	465	opObj.start()	158	opObj.start()
	466	self.~~opObj~~ = ~~opObj~~	159	self.type = 'external'
			160	else:
			161	opObj = className()
	467		162
			163	self.object = opObj
	468	return opObj	164	return opObj
	469		165
	470	class ProcUnitConf():	166	class ProcUnitConf(ConfBase):
	471		167
	472	ELEMENTNAME = 'ProcUnit'	168	ELEMENTNAME = 'ProcUnit'
			169	xml_labels = ['id', 'inputId', 'name']
	473		170
	474	def __init__(self):	171	def setup(self, project_id, id, name, datatype, inputId, err_queue):
	475
	476	self.id = None
	477	self.datatype = None
	478	self.name = None
	479	self.inputId = None
	480	self.opConfObjList = []
	481	self.procUnitObj = None
	482	self.opObjDict = {}
	483
	484	def __getPriority(self):
	485
	486	return len(self.opConfObjList) + 1
	487
	488	def __getNewId(self):
	489
	490	return int(self.id) * 10 + len(self.opConfObjList) + 1
	491
	492	def getElementName(self):
	493
	494	return self.ELEMENTNAME
	495
	496	def getId(self):
	497
	498	return self.id
	499
	500	def updateId(self, new_id):
	501	'''
	502	new_id = int(parentId) * 10 + (int(self.id) % 10)
	503	new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
	504
	505	# If this proc unit has not inputs
	506	#if self.inputId == '0':
	507	#new_inputId = 0
	508
	509	n = 1
	510	for opConfObj in self.opConfObjList:
	511
	512	idOp = str(int(new_id) * 10 + n)
	513	opConfObj.updateId(idOp)
	514
	515	n += 1
	516
	517	self.parentId = str(parentId)
	518	self.id = str(new_id)
	519	#self.inputId = str(new_inputId)
	520	'''
	521	n = 1
	522
	523	def getInputId(self):
	524
	525	return self.inputId
	526
	527	def getOperationObjList(self):
	528
	529	return self.opConfObjList
	530
	531	def getOperationObj(self, name=None):
	532
	533	for opConfObj in self.opConfObjList:
	534
	535	if opConfObj.name != name:
	536	continue
	537
	538	return opConfObj
	539
	540	return None
	541
	542	def getOpObjfromParamValue(self, value=None):
	543
	544	for opConfObj in self.opConfObjList:
	545	if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
	546	continue
	547	return opConfObj
	548	return None
	549
	550	def getProcUnitObj(self):
	551
	552	return self.procUnitObj
	553
	554	def setup(self, project_id, id, name, datatype, inputId, err_queue, lock):
	555	'''	172	'''
	556	id sera el topico a publicar
	557	inputId sera el topico a subscribirse
	558	'''	173	'''
	559		174
	560	# Compatible with old signal chain version
	561	if datatype == None and name == None:	175	if datatype == None and name == None:
	562	raise ValueError('datatype or name should be defined')	176	raise ValueError('datatype or name should be defined')
	563		177
	564	#Definir una condicion para inputId cuando sea 0
	565
	566	if name == None:	178	if name == None:
	567	if 'Proc' in datatype:	179	if 'Proc' in datatype:
	568	name = datatype	180	name = datatype
	569	else:	181	else:
	570	name = '%sProc' % (datatype)	182	name = '%sProc' % (datatype)
	571		183
	572	if datatype == None:	184	if datatype == None:
	573	datatype = name.replace('Proc', '')	185	datatype = name.replace('Proc', '')
	574		186
	575	self.id = str(id)	187	self.id = str(id)
	576	self.project_id = project_id	188	self.project_id = project_id
	577	self.name = name	189	self.name = name
	578	self.datatype = datatype	190	self.datatype = datatype
	579	self.inputId = inputId	191	self.inputId = inputId
	580	self.err_queue = err_queue	192	self.err_queue = err_queue
	581	self.~~lock~~ = ~~lock~~	193	self.operations = []
	582	self.~~opConfObjList~~ = []	194	self.parameters = {}
	583
	584	self.addOperation(name='run', optype='self')
	585
	586	def removeOperations(self):
	587
	588	for obj in self.opConfObjList:
	589	del obj
	590		195
	591	self.opConfObjList = []	196	def removeOperation(self, id):
	592	self.addOperation(name='run')
	593		197
	594	def addParameter(self, **kwargs):	198	i = [1 if x.id==id else 0 for x in self.operations]
	595	'''	199	self.operations.pop(i.index(1))
	596	Add parameters to 'run' operation
	597	'''
	598	opObj = self.opConfObjList[0]
	599		200
	600	opObj.addParameter(**kwargs)	201	def getOperation(self, id):
	601		202
	602	return opObj	203	for conf in self.operations:
			204	if conf.id == id:
			205	return conf
	603		206
	604	def addOperation(self, name, optype='self'):	207	def addOperation(self, name, optype='self'):
	605	'''	208	'''
	606	Actualizacion - > proceso comunicacion
	607	En el caso de optype='self', elminar. DEfinir comuncacion IPC -> Topic
	608	definir el tipoc de socket o comunicacion ipc++
	609
	610	'''	209	'''
	611		210
	612	id = self.__getNewId()	211	id = self.getNewId()
	613	priority = self.__getPriority() # Sin mucho sentido, pero puede usarse	212	conf = OperationConf()
	614	opConfObj = OperationConf()	213	conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
	615	opConfObj.setup(id, name=name, priority=priority, type=optype, project_id=self.project_id, err_queue=self.err_queue, lock=self.lock)	214	self.operations.append(conf)
	616	self.opConfObjList.append(opConfObj)
	617
	618	return opConfObj
	619		215
	620	def makeXml(self, projectElement):	216	return conf
	621		217
	622	procUnitElement = SubElement(projectElement, self.ELEMENTNAME)	218	def readXml(self, element, project_id, err_queue):
	623	procUnitElement.set('id', str(self.id))
	624	procUnitElement.set('name', self.name)
	625	procUnitElement.set('datatype', self.datatype)
	626	procUnitElement.set('inputId', str(self.inputId))
	627		219
	628	for opConfObj in self.opConfObjList:	220	self.id = element.get('id')
	629	opConfObj.makeXml(procUnitElement)	221	self.name = element.get('name')
	630		222	self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
	631	def readXml(self, upElement, project_id):	223	self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
	632
	633	self.id = upElement.get('id')
	634	self.name = upElement.get('name')
	635	self.datatype = upElement.get('datatype')
	636	self.inputId = upElement.get('inputId')
	637	self.project_id = str(project_id)	224	self.project_id = str(project_id)
			225	self.err_queue = err_queue
			226	self.operations = []
			227	self.parameters = {}
	638		228
	639	if self.ELEMENTNAME == 'ReadUnit':	229	for elm in element:
	640	self.datatype = self.datatype.replace('Reader', '')	230	if elm.tag == 'Parameter':
	641		231	self.addParameter(elm.get('name'), elm.get('value'))
	642	if self.ELEMENTNAME == 'ProcUnit':	232	elif elm.tag == 'Operation':
	643	self.datatype = self.datatype.replace('Proc', '')	233	conf = OperationConf()
	644		234	conf.readXml(elm, project_id, err_queue)
	645	if self.inputId == 'None':	235	self.operations.append(conf)
	646	self.inputId = '0'
	647
	648	self.opConfObjList = []
	649
	650	opElementList = upElement.iter(OperationConf().getElementName())
	651
	652	for opElement in opElementList:
	653	opConfObj = OperationConf()
	654	opConfObj.readXml(opElement, project_id)
	655	self.opConfObjList.append(opConfObj)
	656
	657	def printattr(self):
	658
	659	print('%s[%s]: name = %s, datatype = %s, inputId = %s, project_id = %s' % (self.ELEMENTNAME,
	660	self.id,
	661	self.name,
	662	self.datatype,
	663	self.inputId,
	664	self.project_id))
	665
	666	for opConfObj in self.opConfObjList:
	667	opConfObj.printattr()
	668
	669	def getKwargs(self):
	670
	671	opObj = self.opConfObjList[0]
	672	kwargs = opObj.getKwargs()
	673
	674	return kwargs
	675		236
	676	def createObjects(self):	237	def createObjects(self):
	677	'''	238	'''
	678	Instancia de unidades de procesamiento.	239	Instancia de unidades de procesamiento.
	679	'''	240	'''
	680		241
	681	className = eval(self.name)	242	className = eval(self.name)
	682	kwargs = self.getKwargs()	243	kwargs = self.getKwargs()
	683	procUnitObj = className(self.id, self.inputId, self.project_id, self.err_queue, self.lock, 'ProcUnit', **kwargs)	244	procUnitObj = className()
			245	procUnitObj.name = self.name
	684	log.success('creating process...', self.name)	246	log.success('creating process...', self.name)
	685		247
	686	for ~~opConfObj~~ in self.op~~ConfObjList~~:	248	for conf in self.operations:
	687		249
	688	if opConfObj.type == 'self' and opConfObj.name == 'run':	250	opObj = conf.createObject()
	689	continue
	690	elif opConfObj.type == 'self':
	691	opObj = getattr(procUnitObj, opConfObj.name)
	692	else:
	693	opObj = opConfObj.createObject()
	694		251
	695	log.success('adding operation: {}, type:{}'.format(	252	log.success('adding operation: {}, type:{}'.format(
	696	~~opConfObj~~.name,	253	conf.name,
	697	~~opConfObj~~.type), self.name)	254	conf.type), self.name)
	698
	699	procUnitObj.addOperation(opConfObj, opObj)
	700
	701	procUnitObj.start()
	702	self.procUnitObj = procUnitObj
	703		255
	704	def close(self):	256	procUnitObj.addOperation(conf, opObj)
	705		257
	706	for opConfObj in self.opConfObjList:	258	self.object = procUnitObj
	707	if opConfObj.type == 'self':
	708	continue
	709
	710	opObj = self.procUnitObj.getOperationObj(opConfObj.id)
	711	opObj.close()
	712		259
	713	self.procUnitObj.close()	260	def run(self):
			261	'''
			262	'''
	714		263
	715	return	264	return self.object.call(**self.getKwargs())
	716		265
	717		266
	718	class ReadUnitConf(ProcUnitConf):	267	class ReadUnitConf(ProcUnitConf):
	719		268
	720	ELEMENTNAME = 'ReadUnit'	269	ELEMENTNAME = 'ReadUnit'
	721		270
	722	def __init__(self):	271	def __init__(self):
	723		272
	724	self.id = None	273	self.id = None
	725	self.datatype = None	274	self.datatype = None
	726	self.name = None	275	self.name = None
	727	self.inputId = None	276	self.inputId = None
	728	self.op~~ConfObjList~~ = []	277	self.operations = []
	729	self.lock = Event()	278	self.parameters = {}
	730	self.lock.set()
	731	self.lock.n = Value('d', 0)
	732
	733	def getElementName(self):
	734
	735	return self.ELEMENTNAME
	736		279
	737	def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',	280	def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
	738	startTime='', endTime='', server=None, **kwargs):	281	startTime='', endTime='', server=None, **kwargs):
	739		282
	740
	741	'''
	742	*****el id del proceso sera el Topico
	743
	744	Adicion de {topic}, si no esta presente -> error
	745	kwargs deben ser trasmitidos en la instanciacion
	746
	747	'''
	748
	749	# Compatible with old signal chain version
	750	if datatype == None and name == None:	283	if datatype == None and name == None:
	751	raise ValueError('datatype or name should be defined')	284	raise ValueError('datatype or name should be defined')
	752	if name == None:	285	if name == None:
	753	if 'Reader' in datatype:	286	if 'Reader' in datatype:
	754	name = datatype	287	name = datatype
	755	datatype = name.replace('Reader','')	288	datatype = name.replace('Reader','')
	756	else:	289	else:
	757	name = '{}Reader'.format(datatype)	290	name = '{}Reader'.format(datatype)
	758	if datatype == None:	291	if datatype == None:
	759	if 'Reader' in name:	292	if 'Reader' in name:
	760	datatype = name.replace('Reader','')	293	datatype = name.replace('Reader','')
	761	else:	294	else:
	762	datatype = name	295	datatype = name
	763	name = '{}Reader'.format(name)	296	name = '{}Reader'.format(name)
	764		297
	765	self.id = id	298	self.id = id
	766	self.project_id = project_id	299	self.project_id = project_id
	767	self.name = name	300	self.name = name
	768	self.datatype = datatype	301	self.datatype = datatype
	769	if path != '':
	770	self.path = os.path.abspath(path)
	771	self.startDate = startDate
	772	self.endDate = endDate
	773	self.startTime = startTime
	774	self.endTime = endTime
	775	self.server = server
	776	self.err_queue = err_queue	302	self.err_queue = err_queue
	777	self.addRunOperation(**kwargs)
	778
	779	def update(self, **kwargs):
	780
	781	if 'datatype' in kwargs:
	782	datatype = kwargs.pop('datatype')
	783	if 'Reader' in datatype:
	784	self.name = datatype
	785	else:
	786	self.name = '%sReader' % (datatype)
	787	self.datatype = self.name.replace('Reader', '')
	788
	789	attrs = ('path', 'startDate', 'endDate',
	790	'startTime', 'endTime')
	791
	792	for attr in attrs:
	793	if attr in kwargs:
	794	setattr(self, attr, kwargs.pop(attr))
	795
	796	self.updateRunOperation(**kwargs)
	797		303
	798	def removeOperations(self):	304	self.addParameter(name='path', value=path)
			305	self.addParameter(name='startDate', value=startDate)
			306	self.addParameter(name='endDate', value=endDate)
			307	self.addParameter(name='startTime', value=startTime)
			308	self.addParameter(name='endTime', value=endTime)
	799		309
	800	for obj in self.opConfObjList:	310	for key, value in kwargs.items():
	801	del obj	311	self.addParameter(name=key, value=value)
	802
	803	self.opConfObjList = []
	804
	805	def addRunOperation(self, **kwargs):
	806
	807	opObj = self.addOperation(name='run', optype='self')
	808
	809	if self.server is None:
	810	opObj.addParameter(
	811	name='datatype', value=self.datatype, format='str')
	812	opObj.addParameter(name='path', value=self.path, format='str')
	813	opObj.addParameter(
	814	name='startDate', value=self.startDate, format='date')
	815	opObj.addParameter(
	816	name='endDate', value=self.endDate, format='date')
	817	opObj.addParameter(
	818	name='startTime', value=self.startTime, format='time')
	819	opObj.addParameter(
	820	name='endTime', value=self.endTime, format='time')
	821
	822	for key, value in list(kwargs.items()):
	823	opObj.addParameter(name=key, value=value,
	824	format=type(value).__name__)
	825	else:
	826	opObj.addParameter(name='server', value=self.server, format='str')
	827
	828	return opObj
	829
	830	def updateRunOperation(self, **kwargs):
	831
	832	opObj = self.getOperationObj(name='run')
	833	opObj.removeParameters()
	834
	835	opObj.addParameter(name='datatype', value=self.datatype, format='str')
	836	opObj.addParameter(name='path', value=self.path, format='str')
	837	opObj.addParameter(
	838	name='startDate', value=self.startDate, format='date')
	839	opObj.addParameter(name='endDate', value=self.endDate, format='date')
	840	opObj.addParameter(
	841	name='startTime', value=self.startTime, format='time')
	842	opObj.addParameter(name='endTime', value=self.endTime, format='time')
	843
	844	for key, value in list(kwargs.items()):
	845	opObj.addParameter(name=key, value=value,
	846	format=type(value).__name__)
	847
	848	return opObj
	849
	850	def readXml(self, upElement, project_id):
	851
	852	self.id = upElement.get('id')
	853	self.name = upElement.get('name')
	854	self.datatype = upElement.get('datatype')
	855	self.project_id = str(project_id) #yong
	856
	857	if self.ELEMENTNAME == 'ReadUnit':
	858	self.datatype = self.datatype.replace('Reader', '')
	859
	860	self.opConfObjList = []
	861
	862	opElementList = upElement.iter(OperationConf().getElementName())
	863
	864	for opElement in opElementList:
	865	opConfObj = OperationConf()
	866	opConfObj.readXml(opElement, project_id)
	867	self.opConfObjList.append(opConfObj)
	868
	869	if opConfObj.name == 'run':
	870	self.path = opConfObj.getParameterValue('path')
	871	self.startDate = opConfObj.getParameterValue('startDate')
	872	self.endDate = opConfObj.getParameterValue('endDate')
	873	self.startTime = opConfObj.getParameterValue('startTime')
	874	self.endTime = opConfObj.getParameterValue('endTime')
	875		312
	876		313
	877	class Project(Process):	314	class Project(Process):
	878		315
	879	ELEMENTNAME = 'Project'	316	ELEMENTNAME = 'Project'
	880		317
	881	def __init__(self):	318	def __init__(self):
	882		319
	883	Process.__init__(self)	320	Process.__init__(self)
	884	self.id = None	321	self.id = None
	885	self.filename = None	322	self.filename = None
	886	self.description = None	323	self.description = None
	887	self.email = None	324	self.email = None
	888	self.alarm = ~~None~~	325	self.alarm = []
	889	self.~~procUnitConfObjDict~~ = {}	326	self.configurations = {}
	890	self.err_queue = Queue()	327	# self.err_queue = Queue()
			328	self.err_queue = None
			329	self.started = False
	891		330
	892	def __getNewId(self):	331	def getNewId(self):
	893		332
	894	idList = list(self.~~procUnitConfObjDict~~.keys())	333	idList = list(self.configurations.keys())
	895	id = int(self.id) * 10	334	id = int(self.id) * 10
	896		335
	897	while True:	336	while True:
	898	id += 1	337	id += 1
	899		338
	900	if str(id) in idList:	339	if str(id) in idList:
	901	continue	340	continue
	902		341
	903	break	342	break
	904		343
	905	return str(id)	344	return str(id)
	906		345
	907	def getElementName(self):
	908
	909	return self.ELEMENTNAME
	910
	911	def getId(self):
	912
	913	return self.id
	914
	915	def updateId(self, new_id):	346	def updateId(self, new_id):
	916		347
	917	self.id = str(new_id)	348	self.id = str(new_id)
	918		349
	919	keyList = list(self.~~procUnitConfObjDict~~.keys())	350	keyList = list(self.configurations.keys())
	920	keyList.sort()	351	keyList.sort()
	921		352
	922	n = 1	353	n = 1
	923	new~~ProcUnitConfObjDict~~ = {}	354	new_confs = {}
	924		355
	925	for procKey in keyList:	356	for procKey in keyList:
	926		357
	927	~~pro~~c~~UnitC~~onf~~Obj~~ = self.~~procUnitConfObjDict~~[procKey]	358	conf = self.configurations[procKey]
	928	idProcUnit = str(int(self.id) * 10 + n)	359	idProcUnit = str(int(self.id) * 10 + n)
	929	~~pro~~c~~UnitC~~onf~~Obj~~.updateId(idProcUnit)	360	conf.updateId(idProcUnit)
	930	new~~ProcUnitConfObjDict~~[idProcUnit] = ~~pro~~c~~UnitC~~onf~~Obj~~	361	new_confs[idProcUnit] = conf
	931	n += 1	362	n += 1
	932		363
	933	self.~~procUnitConfObjDict~~ = new~~ProcUnitConfObjDict~~	364	self.configurations = new_confs
	934		365
	935	def setup(self, id=1, name='', description='', email=None, alarm=[]):	366	def setup(self, id=1, name='', description='', email=None, alarm=[]):
	936		367
	937	print(' ')
	938	print('' 60)
	939	print('* Starting SIGNAL CHAIN PROCESSING (Multiprocessing) v%s *' % schainpy.__version__)
	940	print('' 60)
	941	print("* Python " + python_version() + " *")
	942	print('' 19)
	943	print(' ')
	944	self.id = str(id)	368	self.id = str(id)
	945	self.description = description	369	self.description = description
	946	self.email = email	370	self.email = email
	947	self.alarm = alarm	371	self.alarm = alarm
	948	if name:	372	if name:
	949	self.name = '{} ({})'.format(Process.__name__, name)	373	self.name = '{} ({})'.format(Process.__name__, name)
	950		374
	951	def update(self, **kwargs):	375	def update(self, **kwargs):
	952		376
	953	for key, value in ~~list~~(kwargs.items()):	377	for key, value in kwargs.items():
	954	setattr(self, key, value)	378	setattr(self, key, value)
	955		379
	956	def clone(self):	380	def clone(self):
	957		381
	958	p = Project()	382	p = Project()
	959	p.procUnitConfObjDict = self.procUnitConfObjDict	383	p.id = self.id
			384	p.name = self.name
			385	p.description = self.description
			386	p.configurations = self.configurations.copy()
			387
	960	return p	388	return p
	961		389
	962	def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):	390	def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
	963		391
	964	'''	392	'''
	965	Actualizacion:
	966	Se agrego un nuevo argumento: topic -relativo a la forma de comunicar los procesos simultaneos
	967
	968	* El id del proceso sera el topico al que se deben subscribir los procUnits para recibir la informacion(data)
	969
	970	'''	393	'''
	971		394
	972	if id is None:	395	if id is None:
	973	idReadUnit = self.__getNewId()	396	idReadUnit = self.getNewId()
	974	else:	397	else:
	975	idReadUnit = str(id)	398	idReadUnit = str(id)
	976		399
	977	~~readUnitConfObj~~ = ReadUnitConf()	400	conf = ReadUnitConf()
	978	~~readUnitConfObj~~.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)	401	conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
	979	self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj	402	self.configurations[conf.id] = conf
	980		403
	981	return ~~readUnitConfObj~~	404	return conf
	982		405
	983	def addProcUnit(self, inputId='0', datatype=None, name=None):	406	def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
	984		407
	985	'''	408	'''
	986	Actualizacion:
	987	Se agrego dos nuevos argumentos: topic_read (lee data de otro procUnit) y topic_write(escribe o envia data a otro procUnit)
	988	Deberia reemplazar a "inputId"
	989
	990	** A fin de mantener el inputID, este sera la representaacion del topicoal que deben subscribirse. El ID propio de la intancia
	991	(proceso) sera el topico de la publicacion, todo sera asignado de manera dinamica.
	992
	993	'''	409	'''
	994		410
	995	idProcUnit = self.__getNewId()	411	if id is None:
	996	procUnitConfObj = ProcUnitConf()	412	idProcUnit = self.getNewId()
	997	input_proc = self.procUnitConfObjDict[inputId]	413	else:
	998	procUnitConfObj.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue, input_proc.lock)	414	idProcUnit = id
	999	self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
	1000
	1001	return procUnitConfObj
	1002
	1003	def removeProcUnit(self, id):
	1004		415
	1005	if id in list(self.procUnitConfObjDict.keys()):	416	conf = ProcUnitConf()
	1006	self.procUnitConfObjDict.pop(id)	417	conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
			418	self.configurations[conf.id] = conf
	1007		419
	1008	def getReadUnitId(self):	420	return conf
	1009		421
	1010	readUnitConfObj = self.getReadUnitObj()	422	def removeProcUnit(self, id):
	1011		423
	1012	return readUnitConfObj.id	424	if id in self.configurations:
			425	self.configurations.pop(id)
	1013		426
	1014	def getReadUnit~~Obj~~(self):	427	def getReadUnit(self):
	1015		428
	1016	for obj in list(self.~~procUnitConfObjDict~~.values()):	429	for obj in list(self.configurations.values()):
	1017	if obj.~~getElementName~~() == 'ReadUnit':	430	if obj.ELEMENTNAME == 'ReadUnit':
	1018	return obj	431	return obj
	1019		432
	1020	return None	433	return None
	1021		434
	1022	def getProcUnit~~Obj~~(self, id=~~None~~, ~~name~~=~~None~~):	435	def getProcUnit(self, id):
	1023
	1024	if id != None:
	1025	return self.procUnitConfObjDict[id]
	1026		436
	1027	if name != None:	437	return self.configurations[id]
	1028	return self.getProcUnitObjByName(name)
	1029		438
	1030	return None	439	def getUnits(self):
	1031		440
	1032	def getProcUnitObjByName(self, name):	441	keys = list(self.configurations)
	1033		442	keys.sort()
	1034	for obj in list(self.procUnitConfObjDict.values()):
	1035	if obj.name == name:
	1036	return obj
	1037		443
	1038	r~~eturn~~ ~~None~~	444	for key in keys:
			445	yield self.configurations[key]
	1039		446
	1040	def procUnitItems(self):	447	def updateUnit(self, id, **kwargs):
	1041		448
	1042	return list(self.procUnitConfObjDict.items())	449	conf = self.configurations[id].update(**kwargs)
	1043		450
	1044	def makeXml(self):	451	def makeXml(self):
	1045		452
	1046	~~projectElement~~ = Element('Project')	453	xml = Element('Project')
	1047	~~projectElement~~.set('id', str(self.id))	454	xml.set('id', str(self.id))
	1048	~~projectElement~~.set('name', self.name)	455	xml.set('name', self.name)
	1049	~~projectElement~~.set('description', self.description)	456	xml.set('description', self.description)
	1050		457
	1051	for ~~pro~~c~~UnitC~~onf~~Obj~~ in ~~list~~(self.~~procUnitConfObjDict~~.values()):	458	for conf in self.configurations.values():
	1052	~~pro~~c~~UnitC~~onf~~Obj~~.makeXml(~~projectElement~~)	459	conf.makeXml(xml)
	1053		460
	1054	self.projectElement = projectElement	461	self.xml = xml
	1055		462
	1056	def writeXml(self, filename=None):	463	def writeXml(self, filename=None):
	1057		464
	1058	if filename == None:	465	if filename == None:
	1059	if self.filename:	466	if self.filename:
	1060	filename = self.filename	467	filename = self.filename
	1061	else:	468	else:
	1062	filename = 'schain.xml'	469	filename = 'schain.xml'
	1063		470
	1064	if not filename:	471	if not filename:
	1065	print('filename has not been defined. Use setFilename(filename) for do it.')	472	print('filename has not been defined. Use setFilename(filename) for do it.')
	1066	return 0	473	return 0
	1067		474
	1068	abs_file = os.path.abspath(filename)	475	abs_file = os.path.abspath(filename)
	1069		476
	1070	if not os.access(os.path.dirname(abs_file), os.W_OK):	477	if not os.access(os.path.dirname(abs_file), os.W_OK):
	1071	print('No write permission on %s' % os.path.dirname(abs_file))	478	print('No write permission on %s' % os.path.dirname(abs_file))
	1072	return 0	479	return 0
	1073		480
	1074	if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):	481	if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
	1075	print('File %s already exists and it could not be overwriten' % abs_file)	482	print('File %s already exists and it could not be overwriten' % abs_file)
	1076	return 0	483	return 0
	1077		484
	1078	self.makeXml()	485	self.makeXml()
	1079		486
	1080	ElementTree(self.~~projectElement~~).write(abs_file, method='xml')	487	ElementTree(self.xml).write(abs_file, method='xml')
	1081		488
	1082	self.filename = abs_file	489	self.filename = abs_file
	1083		490
	1084	return 1	491	return 1
	1085		492
	1086	def readXml(self, filename=~~None~~):	493	def readXml(self, filename):
	1087
	1088	if not filename:
	1089	print('filename is not defined')
	1090	return 0
	1091		494
	1092	abs_file = os.path.abspath(filename)	495	abs_file = os.path.abspath(filename)
	1093		496
	1094	if not os.path.isfile(abs_file):	497	self.configurations = {}
	1095	print('%s file does not exist' % abs_file)
	1096	return 0
	1097
	1098	self.projectElement = None
	1099	self.procUnitConfObjDict = {}
	1100		498
	1101	try:	499	try:
	1102	self.~~projectElement~~ = ElementTree().parse(abs_file)	500	self.xml = ElementTree().parse(abs_file)
	1103	except:	501	except:
	1104	~~print~~('Error reading %s, verify file format' % filename)	502	log.error('Error reading %s, verify file format' % filename)
	1105	return 0	503	return 0
	1106		504
	1107	self.project = self.projectElement.tag	505	self.id = self.xml.get('id')
	1108		506	self.name = self.xml.get('name')
	1109	self.id = self.~~projectElement~~.get('id')	507	self.description = self.xml.get('description')
	1110	self.name = self.projectElement.get('name')	508
	1111	self.description = self.projectElement.get('description')	509	for element in self.xml:
	1112		510	if element.tag == 'ReadUnit':
	1113	readUnitElementList = self.projectElement.iter(	511	conf = ReadUnitConf()
	1114	ReadUnitConf().getElementName())	512	conf.readXml(element, self.id, self.err_queue)
	1115		513	self.configurations[conf.id] = conf
	1116	for readUnitElement in readUnitElementList:	514	elif element.tag == 'ProcUnit':
	1117	~~readUnitConfObj~~ = ~~Read~~UnitConf()	515	conf = ProcUnitConf()
	1118	readUnitConfObj.readXml(readUnitElement, self.id)	516	input_proc = self.configurations[element.get('inputId')]
	1119	self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj	517	conf.readXml(element, self.id, self.err_queue)
	1120		518	self.configurations[conf.id] = conf
	1121	procUnitElementList = self.projectElement.iter(
	1122	ProcUnitConf().getElementName())
	1123
	1124	for procUnitElement in procUnitElementList:
	1125	procUnitConfObj = ProcUnitConf()
	1126	procUnitConfObj.readXml(procUnitElement, self.id)
	1127	self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
	1128		519
	1129	self.filename = abs_file	520	self.filename = abs_file
	1130		521
	1131	return 1	522	return 1
	1132		523
	1133	def __str__(self):	524	def __str__(self):
	1134		525
	1135	~~print~~('Project~~: name =~~ %s, ~~description =~~ %s, ~~id =~~ %s' % (	526	text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
			527	self.id,
	1136	self.name,	528	self.name,
	1137	self.description,	529	self.description,
	1138	self.id))	530	)
	1139		531
	1140	for ~~pro~~c~~UnitC~~onf~~Obj~~ in self.~~procUnitConfObjDict~~.values():	532	for conf in self.configurations.values():
	1141	print(procUnitConfObj)	533	text += '{}'.format(conf)
	1142		534
	1143	def createObjects(self):	535	return text
	1144		536
			537	def createObjects(self):
	1145		538
	1146	keys = list(self.~~procUnitConfObjDict~~.keys())	539	keys = list(self.configurations.keys())
	1147	keys.sort()	540	keys.sort()
	1148	for key in keys:	541	for key in keys:
	1149	self.procUnitConfObjDict[key].createObjects()	542	conf = self.configurations[key]
			543	conf.createObjects()
			544	if conf.inputId is not None:
			545	conf.object.setInput(self.configurations[conf.inputId].object)
	1150		546
	1151	def monitor(self):	547	def monitor(self):
	1152		548
	1153	t = Thread(target=self.__monitor, args=(self.err_queue, self.ctx))	549	t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
	1154	t.start()	550	t.start()
	1155		551
	1156	def __monitor(self, queue, ctx):	552	def _monitor(self, queue, ctx):
	1157		553
	1158	import socket	554	import socket
	1159		555
	1160	procs = 0	556	procs = 0
	1161	err_msg = ''	557	err_msg = ''
	1162		558
	1163	while True:	559	while True:
	1164	msg = queue.get()	560	msg = queue.get()
	1165	if '#_start_#' in msg:	561	if '#_start_#' in msg:
	1166	procs += 1	562	procs += 1
	1167	elif '#_end_#' in msg:	563	elif '#_end_#' in msg:
	1168	procs -=1	564	procs -=1
	1169	else:	565	else:
	1170	err_msg = msg	566	err_msg = msg
	1171		567
	1172	if procs == 0 or 'Traceback' in err_msg:	568	if procs == 0 or 'Traceback' in err_msg:
	1173	break	569	break
	1174	time.sleep(0.1)	570	time.sleep(0.1)
	1175		571
	1176	if '\|' in err_msg:	572	if '\|' in err_msg:
	1177	name, err = err_msg.split('\|')	573	name, err = err_msg.split('\|')
	1178	if 'SchainWarning' in err:	574	if 'SchainWarning' in err:
	1179	log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)	575	log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
	1180	elif 'SchainError' in err:	576	elif 'SchainError' in err:
	1181	log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)	577	log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
	1182	else:	578	else:
	1183	log.error(err, name)	579	log.error(err, name)
	1184	else:	580	else:
	1185	name, err = self.name, err_msg	581	name, err = self.name, err_msg
	1186		582
	1187	time.sleep(2)	583	time.sleep(1)
	1188
	1189	for conf in self.procUnitConfObjDict.values():
	1190	for confop in conf.opConfObjList:
	1191	if confop.type == 'external':
	1192	confop.opObj.terminate()
	1193	conf.procUnitObj.terminate()
	1194		584
	1195	ctx.term()	585	ctx.term()
	1196		586
	1197	message = ''.join(err)	587	message = ''.join(err)
	1198		588
	1199	if err_msg:	589	if err_msg:
	1200	subject = 'SChain v%s: Error running %s\n' % (	590	subject = 'SChain v%s: Error running %s\n' % (
	1201	schainpy.__version__, self.name)	591	schainpy.__version__, self.name)
	1202		592
	1203	subtitle = 'Hostname: %s\n' % socket.gethostbyname(	593	subtitle = 'Hostname: %s\n' % socket.gethostbyname(
	1204	socket.gethostname())	594	socket.gethostname())
	1205	subtitle += 'Working directory: %s\n' % os.path.abspath('./')	595	subtitle += 'Working directory: %s\n' % os.path.abspath('./')
	1206	subtitle += 'Configuration file: %s\n' % self.filename	596	subtitle += 'Configuration file: %s\n' % self.filename
	1207	subtitle += 'Time: %s\n' % str(datetime.datetime.now())	597	subtitle += 'Time: %s\n' % str(datetime.datetime.now())
	1208		598
	1209	readUnitConfObj = self.getReadUnit~~Obj~~()	599	readUnitConfObj = self.getReadUnit()
	1210	if readUnitConfObj:	600	if readUnitConfObj:
	1211	subtitle += '\nInput parameters:\n'	601	subtitle += '\nInput parameters:\n'
	1212	subtitle += '[Data path = %s]\n' % readUnitConfObj.path	602	subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
	1213	subtitle += '[~~Data typ~~e = %s]\n' % readUnitConfObj.~~datatype~~	603	subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
	1214	subtitle += '[~~Start~~ date = %s]\n' % readUnitConfObj.~~startDate~~	604	subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
	1215	subtitle += '[~~End dat~~e = %s]\n' % readUnitConfObj.~~endDate~~	605	subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
	1216	subtitle += '[~~Start~~ time = %s]\n' % readUnitConfObj.~~startTime~~	606	subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
	1217	subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
	1218		607
	1219	a = Alarm(	608	a = Alarm(
	1220	modes=self.alarm,	609	modes=self.alarm,
	1221	email=self.email,	610	email=self.email,
	1222	message=message,	611	message=message,
	1223	subject=subject,	612	subject=subject,
	1224	subtitle=subtitle,	613	subtitle=subtitle,
	1225	filename=self.filename	614	filename=self.filename
	1226	)	615	)
	1227		616
	1228	a.start()	617	a.start()
	1229		618
	1230	def isPaused(self):
	1231	return 0
	1232
	1233	def isStopped(self):
	1234	return 0
	1235
	1236	def runController(self):
	1237	'''
	1238	returns 0 when this process has been stopped, 1 otherwise
	1239	'''
	1240
	1241	if self.isPaused():
	1242	print('Process suspended')
	1243
	1244	while True:
	1245	time.sleep(0.1)
	1246
	1247	if not self.isPaused():
	1248	break
	1249
	1250	if self.isStopped():
	1251	break
	1252
	1253	print('Process reinitialized')
	1254
	1255	if self.isStopped():
	1256	print('Process stopped')
	1257	return 0
	1258
	1259	return 1
	1260
	1261	def setFilename(self, filename):	619	def setFilename(self, filename):
	1262		620
	1263	self.filename = filename	621	self.filename = filename
	1264		622
	1265	def ~~setProxy~~(self):	623	def runProcs(self):
	1266		624
	1267	if not os.path.exists('/tmp/schain'):	625	err = False
	1268	os.mkdir('/tmp/schain')	626	n = len(self.configurations)
	1269		627
	1270	self.ctx = zmq.Context()	628	while not err:
	1271	xpub = self.ctx.socket(zmq.XPUB)	629	for conf in self.getUnits():
	1272	xpub.bind('ipc:///tmp/schain/{}_pub'.format(self.id))	630	ok = conf.run()
	1273	xsub = self.ctx.socket(zmq.XSUB)	631	if ok is 'Error':
	1274	xsub.bind('ipc:///tmp/schain/{}_sub'.format(self.id))	632	n -= 1
	1275	self.monitor()	633	continue
	1276	try:	634	elif not ok:
	1277	zmq.proxy(xpub, xsub)	635	break
	1278	except zmq.ContextTerminated:	636	if n == 0:
	1279	xpub.close()	637	err = True
	1280	xsub.close()
	1281		638
	1282	def run(self):	639	def run(self):
	1283		640
	1284	log.success('Starting {}: {}'.format(self.name, self.id), tag='')	641	log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
			642	self.started = True
	1285	self.start_time = time.time()	643	self.start_time = time.time()
	1286	self.createObjects()	644	self.createObjects()
	1287	self.~~setProxy~~()	645	self.runProcs()
	1288	log.success('{} Done (Time: {}s)'.format(	646	log.success('{} Done (Time: {:4.2f}s)'.format(
	1289	self.name,	647	self.name,
	1290	time.time()-self.start_time), '')	648	time.time()-self.start_time), '')

schainpy/model/io/bltrIO_param.py +1 -1

             '''
             Created on Nov 9, 2016
             @author: roj- LouVD
             '''
             import os
             import sys
             import time
             import glob
             import datetime
             import numpy
             import schainpy.admin
             from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator
             from schainpy.model.data.jrodata import Parameters
             from schainpy.model.io.jroIO_base import JRODataReader, isNumber
             from schainpy.utils import log
             FILE_HEADER_STRUCTURE = numpy.dtype([
                 ('FMN', '<u4'),
                 ('nrec', '<u4'),
                 ('fr_offset', '<u4'),
                 ('id', '<u4'),
                 ('site', 'u1', (32,))
             ])
             REC_HEADER_STRUCTURE = numpy.dtype([
                 ('rmn', '<u4'),
                 ('rcounter', '<u4'),
                 ('nr_offset', '<u4'),
                 ('tr_offset', '<u4'),
                 ('time', '<u4'),
                 ('time_msec', '<u4'),
                 ('tag', 'u1', (32,)),
                 ('comments', 'u1', (32,)),
                 ('lat', '<f4'),
                 ('lon', '<f4'),
                 ('gps_status', '<u4'),
                 ('freq', '<u4'),
                 ('freq0', '<u4'),
                 ('nchan', '<u4'),
                 ('delta_r', '<u4'),
                 ('nranges', '<u4'),
                 ('r0', '<u4'),
                 ('prf', '<u4'),
                 ('ncoh', '<u4'),
                 ('npoints', '<u4'),
                 ('polarization', '<i4'),
                 ('rx_filter', '<u4'),
                 ('nmodes', '<u4'),
                 ('dmode_index', '<u4'),
                 ('dmode_rngcorr', '<u4'),
                 ('nrxs', '<u4'),
                 ('acf_length', '<u4'),
                 ('acf_lags', '<u4'),
                 ('sea_to_atmos', '<f4'),
                 ('sea_notch', '<u4'),
                 ('lh_sea', '<u4'),
                 ('hh_sea', '<u4'),
                 ('nbins_sea', '<u4'),
                 ('min_snr', '<f4'),
                 ('min_cc', '<f4'),
                 ('max_time_diff', '<f4')
             ])
             DATA_STRUCTURE = numpy.dtype([
                 ('range', '<u4'),
                 ('status', '<u4'),
                 ('zonal', '<f4'),
                 ('meridional', '<f4'),
                 ('vertical', '<f4'),
                 ('zonal_a', '<f4'),
                 ('meridional_a', '<f4'),
                 ('corrected_fading', '<f4'),  # seconds
                 ('uncorrected_fading', '<f4'),  # seconds
                 ('time_diff', '<f4'),
                 ('major_axis', '<f4'),
                 ('axial_ratio', '<f4'),
                 ('orientation', '<f4'),
                 ('sea_power', '<u4'),
                 ('sea_algorithm', '<u4')
             ])
-            @MPDecorator
             class BLTRParamReader(JRODataReader, ProcessingUnit):
                 '''
                 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR
                 from *.sswma files
                 '''
                 ext = '.sswma'
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                     self.counter_records = 0
                     self.flagNoMoreFiles = 0
                     self.isConfig = False
                     self.filename = None
                 def setup(self,
                           path=None,
                           startDate=None,
                           endDate=None,
                           ext=None,
                           startTime=datetime.time(0, 0, 0),
                           endTime=datetime.time(23, 59, 59),
                           timezone=0,
                           status_value=0,
                           **kwargs):
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.status_value = status_value
                     self.datatime = datetime.datetime(1900,1,1)
                     self.delay = kwargs.get('delay', 10)
                     self.online = kwargs.get('online', False)
                     self.nTries = kwargs.get('nTries', 3)
                     if self.path is None:
                         raise ValueError("The path is not valid")
                     if ext is None:
                         ext = self.ext
                     self.fileList = self.search_files(self.path, startDate, endDate, ext)
                     self.timezone = timezone
                     self.fileIndex = 0
                     if not self.fileList:
                         raise Warning("There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' " % (
                             path))
                     self.setNextFile()
                 def search_last_file(self):
                     '''
                     Get last file and add it to the list
                     '''
                     for n in range(self.nTries+1):
                         if n>0:
                             log.warning(
                                 "Waiting %0.2f seconds for the next file, try %03d ..." % (self.delay, n+1),
                                 self.name
                             )
                             time.sleep(self.delay)
                         file_list = os.listdir(self.path)
                         file_list.sort()
                         if file_list:
                             if self.filename:
                                 if file_list[-1] not in self.filename:
                                     return file_list[-1]
                                 else:
                                     continue
                             return file_list[-1]
                     return 0
                 def search_files(self, path, startDate, endDate, ext):
                     '''
                      Searching for BLTR rawdata file in path
                      Creating a list of file to proces included in [startDate,endDate]
                      Input:
                          path - Path to find BLTR rawdata files
                          startDate - Select file from this date
                          enDate - Select file until this date
                          ext - Extension of the file to read
                     '''
                     log.success('Searching files in {} '.format(path), 'BLTRParamReader')
                     foldercounter = 0
                     fileList0 = glob.glob1(path, "*%s" % ext)
                     fileList0.sort()
                     for thisFile in fileList0:
                         year = thisFile[-14:-10]
                         if not isNumber(year):
                             continue
                         month = thisFile[-10:-8]
                         if not isNumber(month):
                             continue
                         day = thisFile[-8:-6]
                         if not isNumber(day):
                             continue
                         year, month, day = int(year), int(month), int(day)
                         dateFile = datetime.date(year, month, day)
                         if (startDate > dateFile) or (endDate < dateFile):
                             continue
                         yield thisFile
                     return
                 def setNextFile(self):
                     if self.online:
                         filename = self.search_last_file()
                         if not filename:
                             self.flagNoMoreFiles = 1
                             return 0
                     else:
                         try:
                             filename = next(self.fileList)
                         except StopIteration:
                             self.flagNoMoreFiles = 1
                             return 0
                     log.success('Opening {}'.format(filename), 'BLTRParamReader')
                     dirname, name = os.path.split(filename)
                     # 'peru2' ---> Piura  -   'peru1' ---> Huancayo or Porcuya
                     self.siteFile = filename.split('.')[0]
                     if self.filename is not None:
                         self.fp.close()
                     self.filename = os.path.join(self.path, filename)
                     self.fp = open(self.filename, 'rb')
                     self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
                     self.nrecords = self.header_file['nrec'][0]
                     self.sizeOfFile = os.path.getsize(self.filename)
                     self.counter_records = 0
                     self.flagIsNewFile = 0
                     self.fileIndex += 1
                     return 1
                 def readNextBlock(self):
                     while True:
                         if not self.online and self.counter_records == self.nrecords:
                             self.flagIsNewFile = 1
                             if not self.setNextFile():
                                 return 0
                         try:
                             pointer = self.fp.tell()
                             self.readBlock()
                         except:
                             if self.online and self.waitDataBlock(pointer, 38512) == 1:
                                 continue
                             else:
                                 if not self.setNextFile():
                                     return 0
                         if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
                            (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
                             log.warning(
                                 'Reading Record No. {}/{} -> {} [Skipping]'.format(
                                     self.counter_records,
                                     self.nrecords,
                                     self.datatime.ctime()),
                                 'BLTRParamReader')
                             continue
                         break
                     log.log('Reading Record No. {} -> {}'.format(
                         self.counter_records,
                         # self.nrecords,
                         self.datatime.ctime()), 'BLTRParamReader')
                     return 1
                 def readBlock(self):
                     pointer = self.fp.tell()
                     header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
                     self.nchannels = int(header_rec['nchan'][0] / 2)
                     self.kchan = header_rec['nrxs'][0]
                     self.nmodes = header_rec['nmodes'][0]
                     self.nranges = header_rec['nranges'][0]
                     self.fp.seek(pointer)
                     self.height = numpy.empty((self.nmodes, self.nranges))
                     self.snr = numpy.empty((self.nmodes, int(self.nchannels), self.nranges))
                     self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
                     self.flagDiscontinuousBlock = 0
                     for mode in range(self.nmodes):
                         self.readHeader()
                         data = self.readData()
                         self.height[mode] = (data[0] - self.correction) / 1000.
                         self.buffer[mode] = data[1]
                         self.snr[mode] = data[2]
                     self.counter_records = self.counter_records + self.nmodes
                     return
                 def readHeader(self):
                     '''
                     RecordHeader of BLTR rawdata file
                     '''
                     header_structure = numpy.dtype(
                         REC_HEADER_STRUCTURE.descr + [
                             ('antenna_coord', 'f4', (2, int(self.nchannels))),
                             ('rx_gains', 'u4', (int(self.nchannels),)),
                             ('rx_analysis', 'u4', (int(self.nchannels),))
                         ]
                     )
                     self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
                     self.lat = self.header_rec['lat'][0]
                     self.lon = self.header_rec['lon'][0]
                     self.delta = self.header_rec['delta_r'][0]
                     self.correction = self.header_rec['dmode_rngcorr'][0]
                     self.imode = self.header_rec['dmode_index'][0]
                     self.antenna = self.header_rec['antenna_coord']
                     self.rx_gains = self.header_rec['rx_gains']
                     self.time = self.header_rec['time'][0]
                     dt = datetime.datetime.utcfromtimestamp(self.time)
                     if dt.date()>self.datatime.date():
                         self.flagDiscontinuousBlock = 1
                     self.datatime = dt
                 def readData(self):
                     '''
                     Reading and filtering data block record of BLTR rawdata file,
                     filtering is according to status_value.
                     Input:
                         status_value - Array data is set to NAN for values that are not
                         equal to status_value
                     '''
                     self.nchannels = int(self.nchannels)
                     data_structure = numpy.dtype(
                         DATA_STRUCTURE.descr + [
                             ('rx_saturation', 'u4', (self.nchannels,)),
                             ('chan_offset', 'u4', (2 * self.nchannels,)),
                             ('rx_amp', 'u4', (self.nchannels,)),
                             ('rx_snr', 'f4', (self.nchannels,)),
                             ('cross_snr', 'f4', (self.kchan,)),
                             ('sea_power_relative', 'f4', (self.kchan,))]
                     )
                     data = numpy.fromfile(self.fp, data_structure, self.nranges)
                     height = data['range']
                     winds = numpy.array(
                         (data['zonal'], data['meridional'], data['vertical']))
                     snr = data['rx_snr'].T
                     winds[numpy.where(winds == -9999.)] = numpy.nan
                     winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
                     snr[numpy.where(snr == -9999.)] = numpy.nan
                     snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
                     snr = numpy.power(10, snr / 10)
                     return height, winds, snr
                 def set_output(self):
                     '''
                     Storing data from databuffer to dataOut object
                     '''
                     self.dataOut.data_SNR = self.snr
                     self.dataOut.height = self.height
                     self.dataOut.data = self.buffer
                     self.dataOut.utctimeInit = self.time
                     self.dataOut.utctime = self.dataOut.utctimeInit
                     self.dataOut.useLocalTime = False
                     self.dataOut.paramInterval = 157
                     self.dataOut.timezone = self.timezone
                     self.dataOut.site = self.siteFile
                     self.dataOut.nrecords = self.nrecords / self.nmodes
                     self.dataOut.sizeOfFile = self.sizeOfFile
                     self.dataOut.lat = self.lat
                     self.dataOut.lon = self.lon
                     self.dataOut.channelList = list(range(self.nchannels))
                     self.dataOut.kchan = self.kchan
                     self.dataOut.delta = self.delta
                     self.dataOut.correction = self.correction
                     self.dataOut.nmodes = self.nmodes
                     self.dataOut.imode = self.imode
                     self.dataOut.antenna = self.antenna
                     self.dataOut.rx_gains = self.rx_gains
                     self.dataOut.flagNoData = False
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                 def getData(self):
                     '''
                     Storing data from databuffer to dataOut object
                     '''
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         raise schainpy.admin.SchainError('No More files to read')
                     if not self.readNextBlock():
                         self.dataOut.flagNoData = True
                         raise schainpy.admin.SchainError('Time for wait new file reach!!!')
                     self.set_output()
                     return 1
   No newline at end of file

schainpy/model/io/bltrIO_spectra.py +1 -1

             import os
             import sys
             import glob
             import fnmatch
             import datetime
             import time
             import re
             import h5py
             import numpy
             import pylab as plb
             from scipy.optimize import curve_fit
             from scipy import asarray as ar, exp
             SPEED_OF_LIGHT = 299792458
             SPEED_OF_LIGHT = 3e8
             from .utils import folder_in_range
             import schainpy.admin
             from schainpy.model.data.jrodata import Spectra
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.utils import log
             from schainpy.model.io.jroIO_base import JRODataReader
             def pol2cart(rho, phi):
                 x = rho * numpy.cos(phi)
                 y = rho * numpy.sin(phi)
                 return(x, y)
             FILE_STRUCTURE = numpy.dtype([  # HEADER 48bytes
                 ('FileMgcNumber', '<u4'),  # 0x23020100
                 ('nFDTdataRecors', '<u4'),
                 ('OffsetStartHeader', '<u4'),
                 ('RadarUnitId', '<u4'),
                 ('SiteName', 'S32'),  # Null terminated
             ])
             class FileHeaderBLTR():
                 def __init__(self, fo):
                     self.fo = fo
                     self.size = 48
                     self.read()
                 def read(self):
                     header = numpy.fromfile(self.fo, FILE_STRUCTURE, 1)
                     self.FileMgcNumber = hex(header['FileMgcNumber'][0])
                     self.nFDTdataRecors = int(header['nFDTdataRecors'][0])
                     self.RadarUnitId = int(header['RadarUnitId'][0])
                     self.OffsetStartHeader = int(header['OffsetStartHeader'][0])
                     self.SiteName = header['SiteName'][0]
                 def write(self, fp):
                     headerTuple = (self.FileMgcNumber,
                                    self.nFDTdataRecors,
                                    self.RadarUnitId,
                                    self.SiteName,
                                    self.size)
                     header = numpy.array(headerTuple, FILE_STRUCTURE)
                     header.tofile(fp)
                     ''' ndarray.tofile(fid, sep, format)    Write array to a file as text or binary (default).
                     fid : file or str
                     An open file object, or a string containing a filename.
                     sep : str
                     Separator between array items for text output. If "" (empty), a binary file is written,
                     equivalent to file.write(a.tobytes()).
                     format : str
                     Format string for text file output. Each entry in the array is formatted to text by
                     first converting it to the closest Python type, and then using "format" % item.
                     '''
                     return 1
             RECORD_STRUCTURE = numpy.dtype([  # RECORD HEADER 180+20N bytes
                 ('RecMgcNumber', '<u4'),  # 0x23030001
                 ('RecCounter', '<u4'),  # Record counter(0,1, ...)
                 # Offset to start of next record form start of this record
                                         ('Off2StartNxtRec', '<u4'),
                                         # Offset to start of data from start of this record
                                         ('Off2StartData', '<u4'),
                                         # Epoch time stamp of start of acquisition (seconds)
                                         ('nUtime', '<i4'),
                                         # Millisecond component of time stamp (0,...,999)
                                         ('nMilisec', '<u4'),
                                         # Experiment tag name (null terminated)
                                         ('ExpTagName', 'S32'),
                                         # Experiment comment (null terminated)
                                         ('ExpComment', 'S32'),
                                         # Site latitude (from GPS) in degrees (positive implies North)
                                         ('SiteLatDegrees', '<f4'),
                                         # Site longitude (from GPS) in degrees (positive implies East)
                                         ('SiteLongDegrees', '<f4'),
                                         # RTC GPS engine status (0=SEEK, 1=LOCK, 2=NOT FITTED, 3=UNAVAILABLE)
                                         ('RTCgpsStatus', '<u4'),
                                         ('TransmitFrec', '<u4'),  # Transmit frequency (Hz)
                                         ('ReceiveFrec', '<u4'),  # Receive frequency
                                         # First local oscillator frequency (Hz)
                                         ('FirstOsciFrec', '<u4'),
                                         # (0="O", 1="E", 2="linear 1", 3="linear2")
                                         ('Polarisation', '<u4'),
                                         # Receiver filter settings (0,1,2,3)
                                         ('ReceiverFiltSett', '<u4'),
                                         # Number of modes in use (1 or 2)
                                         ('nModesInUse', '<u4'),
                                         # Dual Mode index number for these data (0 or 1)
                                         ('DualModeIndex', '<u4'),
                                         # Dual Mode range correction for these data (m)
                                         ('DualModeRange', '<u4'),
                                         # Number of digital channels acquired (2*N)
                                         ('nDigChannels', '<u4'),
                                         # Sampling resolution (meters)
                                         ('SampResolution', '<u4'),
                                         # Number of range gates sampled
                                         ('nHeights', '<u4'),
                                         # Start range of sampling (meters)
                                         ('StartRangeSamp', '<u4'),
                                         ('PRFhz', '<u4'),  # PRF (Hz)
                                         ('nCohInt', '<u4'),  # Integrations
                                         # Number of data points transformed
                                         ('nProfiles', '<u4'),
                                         # Number of receive beams stored in file (1 or N)
                                         ('nChannels', '<u4'),
                                         ('nIncohInt', '<u4'),  # Number of spectral averages
                                         # FFT windowing index (0 = no window)
                                         ('FFTwindowingInd', '<u4'),
                                         # Beam steer angle (azimuth) in degrees (clockwise from true North)
                                         ('BeamAngleAzim', '<f4'),
                                         # Beam steer angle (zenith) in degrees (0=> vertical)
                                         ('BeamAngleZen', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaCoord0', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaAngl0', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaCoord1', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaAngl1', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaCoord2', '<f4'),
                                         # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
                                         ('AntennaAngl2', '<f4'),
                                         # Receiver phase calibration (degrees) - N values
                                         ('RecPhaseCalibr0', '<f4'),
                                         # Receiver phase calibration (degrees) - N values
                                         ('RecPhaseCalibr1', '<f4'),
                                         # Receiver phase calibration (degrees) - N values
                                         ('RecPhaseCalibr2', '<f4'),
                                         # Receiver amplitude calibration (ratio relative to receiver one) - N values
                                         ('RecAmpCalibr0', '<f4'),
                                         # Receiver amplitude calibration (ratio relative to receiver one) - N values
                                         ('RecAmpCalibr1', '<f4'),
                                         # Receiver amplitude calibration (ratio relative to receiver one) - N values
                                         ('RecAmpCalibr2', '<f4'),
                                         # Receiver gains in dB - N values
                                         ('ReceiverGaindB0', '<i4'),
                                         # Receiver gains in dB - N values
                                         ('ReceiverGaindB1', '<i4'),
                                         # Receiver gains in dB - N values
                                         ('ReceiverGaindB2', '<i4'),
             ])
             class RecordHeaderBLTR():
                 def __init__(self, fo):
                     self.fo = fo
                     self.OffsetStartHeader = 48
                     self.Off2StartNxtRec = 811248
                 def read(self, block):
                     OffRHeader = self.OffsetStartHeader + block * self.Off2StartNxtRec
                     self.fo.seek(OffRHeader, os.SEEK_SET)
                     header = numpy.fromfile(self.fo, RECORD_STRUCTURE, 1)
                     self.RecMgcNumber = hex(header['RecMgcNumber'][0])  # 0x23030001
                     self.RecCounter = int(header['RecCounter'][0])
                     self.Off2StartNxtRec = int(header['Off2StartNxtRec'][0])
                     self.Off2StartData = int(header['Off2StartData'][0])
                     self.nUtime = header['nUtime'][0]
                     self.nMilisec = header['nMilisec'][0]
                     self.ExpTagName = '' # str(header['ExpTagName'][0])
                     self.ExpComment = '' # str(header['ExpComment'][0])
                     self.SiteLatDegrees = header['SiteLatDegrees'][0]
                     self.SiteLongDegrees = header['SiteLongDegrees'][0]
                     self.RTCgpsStatus = header['RTCgpsStatus'][0]
                     self.TransmitFrec = header['TransmitFrec'][0]
                     self.ReceiveFrec = header['ReceiveFrec'][0]
                     self.FirstOsciFrec = header['FirstOsciFrec'][0]
                     self.Polarisation = header['Polarisation'][0]
                     self.ReceiverFiltSett = header['ReceiverFiltSett'][0]
                     self.nModesInUse = header['nModesInUse'][0]
                     self.DualModeIndex = header['DualModeIndex'][0]
                     self.DualModeRange = header['DualModeRange'][0]
                     self.nDigChannels = header['nDigChannels'][0]
                     self.SampResolution = header['SampResolution'][0]
                     self.nHeights = header['nHeights'][0]
                     self.StartRangeSamp = header['StartRangeSamp'][0]
                     self.PRFhz = header['PRFhz'][0]
                     self.nCohInt = header['nCohInt'][0]
                     self.nProfiles = header['nProfiles'][0]
                     self.nChannels = header['nChannels'][0]
                     self.nIncohInt = header['nIncohInt'][0]
                     self.FFTwindowingInd = header['FFTwindowingInd'][0]
                     self.BeamAngleAzim = header['BeamAngleAzim'][0]
                     self.BeamAngleZen = header['BeamAngleZen'][0]
                     self.AntennaCoord0 = header['AntennaCoord0'][0]
                     self.AntennaAngl0 = header['AntennaAngl0'][0]
                     self.AntennaCoord1 = header['AntennaCoord1'][0]
                     self.AntennaAngl1 = header['AntennaAngl1'][0]
                     self.AntennaCoord2 = header['AntennaCoord2'][0]
                     self.AntennaAngl2 = header['AntennaAngl2'][0]
                     self.RecPhaseCalibr0 = header['RecPhaseCalibr0'][0]
                     self.RecPhaseCalibr1 = header['RecPhaseCalibr1'][0]
                     self.RecPhaseCalibr2 = header['RecPhaseCalibr2'][0]
                     self.RecAmpCalibr0 = header['RecAmpCalibr0'][0]
                     self.RecAmpCalibr1 = header['RecAmpCalibr1'][0]
                     self.RecAmpCalibr2 = header['RecAmpCalibr2'][0]
                     self.ReceiverGaindB0 = header['ReceiverGaindB0'][0]
                     self.ReceiverGaindB1 = header['ReceiverGaindB1'][0]
                     self.ReceiverGaindB2 = header['ReceiverGaindB2'][0]
                     self.ipp = 0.5 * (SPEED_OF_LIGHT / self.PRFhz)
                     self.RHsize = 180 + 20 * self.nChannels
                     self.Datasize = self.nProfiles * self.nChannels * self.nHeights * 2 * 4
                     endFp = self.OffsetStartHeader + self.RecCounter * self.Off2StartNxtRec
                     if OffRHeader > endFp:
                         sys.stderr.write(
                             "Warning %s: Size value read from System Header is lower than it has to be\n" % fp)
                         return 0
                     if OffRHeader < endFp:
                         sys.stderr.write(
                             "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp)
                         return 0
                     return 1
-            @MPDecorator
             class BLTRSpectraReader (ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.ext = ".fdt"
                     self.optchar = "P"
                     self.fpFile = None
                     self.fp = None
                     self.BlockCounter = 0
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.fileSelector = 0
                     self.Off2StartNxtRec = 0
                     self.RecCounter = 0
                     self.flagNoMoreFiles = 0
                     self.data_spc = None
                     self.data_cspc = None
                     self.path = None
                     self.OffsetStartHeader = 0
                     self.Off2StartData = 0
                     self.ipp = 0
                     self.nFDTdataRecors = 0
                     self.blocksize = 0
                     self.dataOut = Spectra()
                     self.dataOut.flagNoData = False
                 def search_files(self):
                     '''
                     Function that indicates the number of .fdt files that exist in the folder to be read.
                     It also creates an organized list with the names of the files to read.
                     '''
                     files = glob.glob(os.path.join(self.path, '*{}'.format(self.ext)))
                     files = sorted(files)
                     for f in files:
                         filename = f.split('/')[-1]
                         if folder_in_range(filename.split('.')[1], self.startDate, self.endDate, '%Y%m%d'):
                             self.filenameList.append(f)
                 def run(self, **kwargs):
                     '''
                     This method will be the one that will initiate the data entry, will be called constantly.
                     You should first verify that your Setup () is set up and then continue to acquire
                     the data to be processed with getData ().
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
                 def setup(self,
                           path=None,
                           startDate=None,
                           endDate=None,
                           startTime=None,
                           endTime=None,
                           walk=True,
                           code=None,
                           online=False,
                           mode=None,
                           **kwargs):
                     self.isConfig = True
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.walk = walk
                     self.mode = int(mode)
                     self.search_files()
                     if self.filenameList:
                         self.readFile()
                 def getData(self):
                     '''
                     Before starting this function, you should check that there is still an unread file,
                     If there are still blocks to read or if the data block is empty.
                     You should call the file "read".
                     '''
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         raise schainpy.admin.SchainError('No more files')
                     self.readBlock()
                 def readFile(self):
                     '''
                     You must indicate if you are reading in Online or Offline mode and load the
                     The parameters for this file reading mode.
                     Then you must do 2 actions:
 . Get the BLTR FileHeader.
 . Start reading the first block.
                     '''
                     if self.fileSelector < len(self.filenameList):
                         log.success('Opening file: {}'.format(self.filenameList[self.fileSelector]), self.name)
                         self.fp = open(self.filenameList[self.fileSelector])
                         self.fheader = FileHeaderBLTR(self.fp)
                         self.rheader = RecordHeaderBLTR(self.fp)
                         self.nFDTdataRecors = self.fheader.nFDTdataRecors
                         self.fileSelector += 1
                         self.BlockCounter = 0
                         return 1
                     else:
                         self.flagNoMoreFiles = True
                         self.dataOut.flagNoData = True
                         return 0
                 def readBlock(self):
                     '''
                     It should be checked if the block has data, if it is not passed to the next file.
                     Then the following is done:
 . Read the RecordHeader
 . Fill the buffer with the current block number.
                     '''
                     if self.BlockCounter == self.nFDTdataRecors:
                         if not self.readFile():
                             return
                     if self.mode == 1:
                         self.rheader.read(self.BlockCounter+1)
                     elif self.mode == 0:
                         self.rheader.read(self.BlockCounter)
                     self.RecCounter = self.rheader.RecCounter
                     self.OffsetStartHeader = self.rheader.OffsetStartHeader
                     self.Off2StartNxtRec = self.rheader.Off2StartNxtRec
                     self.Off2StartData = self.rheader.Off2StartData
                     self.nProfiles = self.rheader.nProfiles
                     self.nChannels = self.rheader.nChannels
                     self.nHeights = self.rheader.nHeights
                     self.frequency = self.rheader.TransmitFrec
                     self.DualModeIndex = self.rheader.DualModeIndex
                     self.pairsList = [(0, 1), (0, 2), (1, 2)]
                     self.dataOut.pairsList = self.pairsList
                     self.nRdPairs = len(self.dataOut.pairsList)
                     self.dataOut.nRdPairs = self.nRdPairs
                     self.dataOut.heightList = (self.rheader.StartRangeSamp + numpy.arange(self.nHeights) * self.rheader.SampResolution) / 1000.
                     self.dataOut.channelList = range(self.nChannels)
                     self.dataOut.nProfiles=self.rheader.nProfiles
                     self.dataOut.nIncohInt=self.rheader.nIncohInt
                     self.dataOut.nCohInt=self.rheader.nCohInt
                     self.dataOut.ippSeconds= 1/float(self.rheader.PRFhz)
                     self.dataOut.PRF=self.rheader.PRFhz
                     self.dataOut.nFFTPoints=self.rheader.nProfiles
                     self.dataOut.utctime = self.rheader.nUtime + self.rheader.nMilisec/1000.
                     self.dataOut.timeZone = 0
                     self.dataOut.useLocalTime = False
                     self.dataOut.nmodes = 2
                     log.log('Reading block {} - {}'.format(self.BlockCounter, self.dataOut.datatime), self.name)
                     OffDATA = self.OffsetStartHeader + self.RecCounter * \
                         self.Off2StartNxtRec + self.Off2StartData
                     self.fp.seek(OffDATA, os.SEEK_SET)
                     self.data_fft = numpy.fromfile(self.fp, [('complex','<c8')], self.nProfiles*self.nChannels*self.nHeights )
                     self.data_fft = self.data_fft.astype(numpy.dtype('complex'))
                     self.data_block = numpy.reshape(self.data_fft,(self.nHeights, self.nChannels, self.nProfiles))
                     self.data_block = numpy.transpose(self.data_block, (1,2,0))
                     copy = self.data_block.copy()
                     spc = copy * numpy.conjugate(copy)
                     self.data_spc = numpy.absolute(spc)  # valor absoluto o magnitud
                     self.dataOut.data_spc = self.data_spc
                     cspc = self.data_block.copy()
                     self.data_cspc = self.data_block.copy()
                     for i in range(self.nRdPairs):
                         chan_index0 = self.dataOut.pairsList[i][0]
                         chan_index1 = self.dataOut.pairsList[i][1]
                         self.data_cspc[i, :, :] = cspc[chan_index0, :, :] * numpy.conjugate(cspc[chan_index1, :, :])
                     '''Getting Eij and Nij'''
                     (AntennaX0, AntennaY0) = pol2cart(
                         self.rheader.AntennaCoord0, self.rheader.AntennaAngl0 * numpy.pi / 180)
                     (AntennaX1, AntennaY1) = pol2cart(
                         self.rheader.AntennaCoord1, self.rheader.AntennaAngl1 * numpy.pi / 180)
                     (AntennaX2, AntennaY2) = pol2cart(
                         self.rheader.AntennaCoord2, self.rheader.AntennaAngl2 * numpy.pi / 180)
                     E01 = AntennaX0 - AntennaX1
                     N01 = AntennaY0 - AntennaY1
                     E02 = AntennaX0 - AntennaX2
                     N02 = AntennaY0 - AntennaY2
                     E12 = AntennaX1 - AntennaX2
                     N12 = AntennaY1 - AntennaY2
                     self.ChanDist = numpy.array(
                         [[E01, N01], [E02, N02], [E12, N12]])
                     self.dataOut.ChanDist = self.ChanDist
                     self.BlockCounter += 2
                     self.dataOut.data_spc = self.data_spc
                     self.dataOut.data_cspc =self.data_cspc

schainpy/model/io/jroIO_base.py +26 -21

             """
             Created on Jul 2, 2014
             @author: roj-idl71
             """
             import os
             import sys
             import glob
             import time
             import numpy
             import fnmatch
             import inspect
             import time
             import datetime
             import zmq
+            from schainpy.model.proc.jroproc_base import Operation
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
             from schainpy.utils import log
             import schainpy.admin
             LOCALTIME = True
             DT_DIRECTIVES = {
                 '%Y': 4,
                 '%y': 2,
                 '%m': 2,
                 '%d': 2,
                 '%j': 3,
                 '%H': 2,
                 '%M': 2,
                 '%S': 2,
                 '%f': 6
             }
             def isNumber(cad):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float(cad)
                     return True
                 except:
                     return False
             def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
                 """
                 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                     endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 try:
                     fp = open(filename, 'rb')
                 except IOError:
                     print("The file %s can't be opened" % (filename))
                     return 0
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 if not(sts):
                     print("Skipping the file %s because it has not a valid header" % (filename))
                     return 0
                 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                     return 0
                 return 1
             def isTimeInRange(thisTime, startTime, endTime):
                 if endTime >= startTime:
                     if (thisTime < startTime) or (thisTime > endTime):
                         return 0
                     return 1
                 else:
                     if (thisTime < startTime) and (thisTime > endTime):
                         return 0
                     return 1
             def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                     startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                     endTime            :    tiempo final del rango seleccionado en formato datetime.time
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 try:
                     fp = open(filename, 'rb')
                 except IOError:
                     print("The file %s can't be opened" % (filename))
                     return None
                 firstBasicHeaderObj = BasicHeader(LOCALTIME)
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
                 processingHeaderObj = ProcessingHeader()
                 lastBasicHeaderObj = BasicHeader(LOCALTIME)
                 sts = firstBasicHeaderObj.read(fp)
                 if not(sts):
                     print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
                     return None
                 if not systemHeaderObj.read(fp):
                     return None
                 if not radarControllerHeaderObj.read(fp):
                     return None
                 if not processingHeaderObj.read(fp):
                     return None
                 filesize = os.path.getsize(filename)
                 offset = processingHeaderObj.blockSize + 24  # header size
                 if filesize <= offset:
                     print("[Reading] %s: This file has not enough data" % filename)
                     return None
                 fp.seek(-offset, 2)
                 sts = lastBasicHeaderObj.read(fp)
                 fp.close()
                 thisDatetime = lastBasicHeaderObj.datatime
                 thisTime_last_block = thisDatetime.time()
                 thisDatetime = firstBasicHeaderObj.datatime
                 thisDate = thisDatetime.date()
                 thisTime_first_block = thisDatetime.time()
                 # General case
                 #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                 #-----------o----------------------------o-----------
                 #       startTime                     endTime
                 if endTime >= startTime:
                     if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
                         return None
                     return thisDatetime
                 # If endTime < startTime then endTime belongs to the next day
                 #<<<<<<<<<<<o                            o>>>>>>>>>>>
                 #-----------o----------------------------o-----------
                 #        endTime                    startTime
                 if (thisDate == startDate) and (thisTime_last_block < startTime):
                     return None
                 if (thisDate == endDate) and (thisTime_first_block > endTime):
                     return None
                 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
                     return None
                 return thisDatetime
             def isFolderInDateRange(folder, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     folder            :    nombre completo del directorio.
                                            Su formato deberia ser "/path_root/?YYYYDDD"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el directorio no tiene el formato adecuado
                 """
                 basename = os.path.basename(folder)
                 if not isRadarFolder(basename):
                     print("The folder %s has not the rigth format" % folder)
                     return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFolder(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def isFileInDateRange(filename, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                                            Su formato deberia ser "?YYYYDDDsss"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                                                 sss     :    set
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no tiene el formato adecuado
                 """
                 basename = os.path.basename(filename)
                 if not isRadarFile(basename):
                     print("The filename %s has not the rigth format" % filename)
                     return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFile(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def getFileFromSet(path, ext, set):
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     try:
                         year = int(thisFile[1:5])
                         doy = int(thisFile[5:8])
                     except:
                         continue
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 myfile = fnmatch.filter(
                     validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
                 if len(myfile) != 0:
                     return myfile[0]
                 else:
                     filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
                     print('the filename %s does not exist' % filename)
                     print('...going to the last file: ')
                 if validFilelist:
                     validFilelist = sorted(validFilelist, key=str.lower)
                     return validFilelist[-1]
                 return None
             def getlastFileFromPath(path, ext):
                 """
                 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     year = thisFile[1:5]
                     if not isNumber(year):
                         continue
                     doy = thisFile[5:8]
                     if not isNumber(doy):
                         continue
                     year = int(year)
                     doy = int(doy)
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 if validFilelist:
                     validFilelist = sorted(validFilelist, key=str.lower)
                     return validFilelist[-1]
                 return None
             def isRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return 0
                 return 1
             def isRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
                 except:
                     return 0
                 return 1
             def getDateFromRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
                 return thisDate
             def getDateFromRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
                 return thisDate
             def parse_format(s, fmt):
                 for i in range(fmt.count('%')):
                     x = fmt.index('%')
                     d = DT_DIRECTIVES[fmt[x:x+2]]
                     fmt = fmt.replace(fmt[x:x+2], s[x:x+d])
                 return fmt
             class Reader(object):
                 c = 3E8
                 isConfig = False
                 dtype = None
                 pathList = []
                 filenameList = []
                 datetimeList = []
                 filename = None
                 ext = None
                 flagIsNewFile = 1
                 flagDiscontinuousBlock = 0
                 flagIsNewBlock = 0
                 flagNoMoreFiles = 0
                 fp = None
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 versionFile = 1103
                 fileSize = None
                 fileSizeByHeader = None
                 fileIndex = -1
                 profileIndex = None
                 blockIndex = 0
                 nTotalBlocks = 0
                 maxTimeStep = 30
                 lastUTTime = None
                 datablock = None
                 dataOut = None
                 getByBlock = False
                 path = None
                 startDate = None
                 endDate = None
                 startTime = datetime.time(0, 0, 0)
                 endTime = datetime.time(23, 59, 59)
                 set = None
                 expLabel = ""
                 online = False
                 delay = 60
                 nTries = 3  # quantity tries
                 nFiles = 3  # number of files for searching
                 walk = True
                 getblock = False
                 nTxs = 1
                 realtime = False
                 blocksize = 0
                 blocktime = None
                 warnings = True
                 verbose = True
                 server = None
                 format = None
                 oneDDict = None
                 twoDDict = None
                 independentParam = None
                 filefmt = None
                 folderfmt = None
                 open_file = open
                 open_mode = 'rb'
                 def run(self):
                     raise NotImplementedError
                 def getAllowedArgs(self):
                     if hasattr(self, '__attrs__'):
                         return self.__attrs__
                     else:
                         return inspect.getargspec(self.run).args
                 def set_kwargs(self, **kwargs):
                     for key, value in kwargs.items():
                         setattr(self, key, value)
                 def find_folders(self, path, startDate, endDate, folderfmt, last=False):
                     folders = [x for f in path.split(',')
                                for x in os.listdir(f) if os.path.isdir(os.path.join(f, x))]
                     folders.sort()
                     if last:
                         folders = [folders[-1]]
                     for folder in folders:
                         try:
                             dt = datetime.datetime.strptime(parse_format(folder, folderfmt), folderfmt).date()
                             if dt >= startDate and dt <= endDate:
                                 yield os.path.join(path, folder)
                             else:
                                 log.log('Skiping folder {}'.format(folder), self.name)
                         except Exception as e:
                             log.log('Skiping folder {}'.format(folder), self.name)
                             continue
                     return
                 def find_files(self, folders, ext, filefmt, startDate=None, endDate=None,
                                expLabel='', last=False):
                     for path in folders:
                         files = glob.glob1(path, '*{}'.format(ext))
                         files.sort()
                         if last:
                             if files:
                                 fo = files[-1]
                                 try:
                                     dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
                                     yield os.path.join(path, expLabel, fo)
                                 except Exception as e:
                                     pass
                                 return
                             else:
                                 return
                         for fo in files:
                             try:
                                 dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
                                 if dt >= startDate and dt <= endDate:
                                     yield os.path.join(path, expLabel, fo)
                                 else:
                                     log.log('Skiping file {}'.format(fo), self.name)
                             except Exception as e:
                                 log.log('Skiping file {}'.format(fo), self.name)
                                 continue
                 def searchFilesOffLine(self, path, startDate, endDate,
                                        expLabel, ext, walk,
                                        filefmt, folderfmt):
                     """Search files in offline mode for the given arguments
                     Return:
                         Generator of files
                     """
                     if walk:
                         folders = self.find_folders(
                             path, startDate, endDate, folderfmt)
                     else:
                         folders = path.split(',')
                     return self.find_files(
                         folders, ext, filefmt, startDate, endDate, expLabel)
                 def searchFilesOnLine(self, path, startDate, endDate,
                                       expLabel, ext, walk,
                                       filefmt, folderfmt):
                     """Search for the last file of the last folder
                     Arguments:
                         path        :    carpeta donde estan contenidos los files que contiene data
                         expLabel    :    Nombre del subexperimento (subfolder)
                         ext         :    extension de los files
                         walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                     Return:
                         generator with the full path of last filename
                     """
                     if walk:
                         folders = self.find_folders(
                             path, startDate, endDate, folderfmt, last=True)
                     else:
                         folders = path.split(',')
                     return self.find_files(
                         folders, ext, filefmt, startDate, endDate, expLabel, last=True)
                 def setNextFile(self):
                     """Set the next file to be readed open it and parse de file header"""
                     while True:
                         if self.fp != None:
                             self.fp.close()
                         if self.online:
                             newFile = self.setNextFileOnline()
                         else:
                             newFile = self.setNextFileOffline()
                         if not(newFile):
                             if self.online:
                                 raise schainpy.admin.SchainError('Time to wait for new files reach')
                             else:
                                 if self.fileIndex == -1:
                                     raise schainpy.admin.SchainWarning('No files found in the given path')
                                 else:
                                     raise schainpy.admin.SchainWarning('No more files to read')
                         if self.verifyFile(self.filename):
                             break
                     log.log('Opening file: %s' % self.filename, self.name)
                     self.readFirstHeader()
                     self.nReadBlocks = 0
                 def setNextFileOnline(self):
                     """Check for the next file to be readed in online mode.
                     Set:
                         self.filename
                         self.fp
                         self.filesize
                     Return:
                         boolean
                     """
                     nextFile = True
                     nextDay = False
                     for nFiles in range(self.nFiles+1):
                         for nTries in range(self.nTries):
                             fullfilename, filename = self.checkForRealPath(nextFile, nextDay)
                             if fullfilename is not None:
                                 break
                             log.warning(
                                 "Waiting %0.2f sec for the next file: \"%s\" , try %02d ..." % (self.delay, filename, nTries + 1),
                                 self.name)
                             time.sleep(self.delay)
                             nextFile = False
                             continue
                         if fullfilename is not None:
                             break
                         self.nTries = 1
                         nextFile = True
                         if nFiles == (self.nFiles - 1):
                             log.log('Trying with next day...', self.name)
                             nextDay = True
                             self.nTries = 3
                     if fullfilename:
                         self.fileSize = os.path.getsize(fullfilename)
                         self.filename = fullfilename
                         self.flagIsNewFile = 1
                         if self.fp != None:
                             self.fp.close()
                         self.fp = self.open_file(fullfilename, self.open_mode)
                         self.flagNoMoreFiles = 0
                         self.fileIndex += 1
                         return 1
                     else:
                         return 0
                 def setNextFileOffline(self):
                     """Open the next file to be readed in offline mode"""
                     try:
                         filename = next(self.filenameList)
                         self.fileIndex +=1
                     except StopIteration:
                         self.flagNoMoreFiles = 1
                         return 0
                     self.filename = filename
                     self.fileSize = os.path.getsize(filename)
                     self.fp = self.open_file(filename, self.open_mode)
                     self.flagIsNewFile = 1
                     return 1
                 @staticmethod
                 def isDateTimeInRange(dt, startDate, endDate, startTime, endTime):
                     """Check if the given datetime is in range"""
                     if startDate <= dt.date() <= endDate:
                         if startTime <= dt.time() <= endTime:
                             return True
                     return False
                 def verifyFile(self, filename):
                     """Check for a valid file
                     Arguments:
                         filename -- full path filename
                     Return:
                         boolean
                     """
                     return True
                 def checkForRealPath(self, nextFile, nextDay):
                     """Check if the next file to be readed exists"""
                     raise NotImplementedError
                 def readFirstHeader(self):
                     """Parse the file header"""
                     pass
             class JRODataReader(Reader):
                 utc = 0
                 nReadBlocks = 0
                 foldercounter = 0
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 __isFirstTimeOnline = 1
-                __printInfo = True
                 filefmt = "*%Y%j***"
                 folderfmt = "*%Y%j"
+                __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'online', 'delay', 'walk']
                 def getDtypeWidth(self):
                     dtype_index = get_dtype_index(self.dtype)
                     dtype_width = get_dtype_width(dtype_index)
                     return dtype_width
                 def checkForRealPath(self, nextFile, nextDay):
                     """Check if the next file to be readed exists.
                     Example    :
                         nombre correcto del file es  .../.../D2009307/P2009307367.ext
                         Entonces la funcion prueba con las siguientes combinaciones
                             .../.../y2009307367.ext
                             .../.../Y2009307367.ext
                             .../.../x2009307/y2009307367.ext
                             .../.../x2009307/Y2009307367.ext
                             .../.../X2009307/y2009307367.ext
                             .../.../X2009307/Y2009307367.ext
                         siendo para este caso, la ultima combinacion de letras, identica al file buscado
                     Return:
                         str -- fullpath of the file
                     """
                     if nextFile:
                         self.set += 1
                     if nextDay:
                         self.set = 0
                         self.doy += 1
                     foldercounter = 0
                     prefixDirList = [None, 'd', 'D']
                     if self.ext.lower() == ".r":  # voltage
                         prefixFileList = ['d', 'D']
                     elif self.ext.lower() == ".pdata":  # spectra
                         prefixFileList = ['p', 'P']
                     # barrido por las combinaciones posibles
                     for prefixDir in prefixDirList:
                         thispath = self.path
                         if prefixDir != None:
                             # formo el nombre del directorio xYYYYDDD (x=d o x=D)
                             if foldercounter == 0:
                                 thispath = os.path.join(self.path, "%s%04d%03d" %
                                                         (prefixDir, self.year, self.doy))
                             else:
                                 thispath = os.path.join(self.path, "%s%04d%03d_%02d" % (
                                     prefixDir, self.year, self.doy, foldercounter))
                         for prefixFile in prefixFileList:  # barrido por las dos combinaciones posibles de "D"
                             # formo el nombre del file xYYYYDDDSSS.ext
                             filename = "%s%04d%03d%03d%s" % (prefixFile, self.year, self.doy, self.set, self.ext)
                             fullfilename = os.path.join(
                                 thispath, filename)
                             if os.path.exists(fullfilename):
                                 return fullfilename, filename
                     return None, filename
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range(self.nTries):
                         self.fp.close()
                         self.fp = open(self.filename, 'rb')
                         self.fp.seek(currentPointer)
                         self.fileSize = os.path.getsize(self.filename)
                         currentSize = self.fileSize - currentPointer
                         if (currentSize >= neededSize):
                             self.basicHeaderObj.read(self.fp)
                             return 1
                         if self.fileSize == self.fileSizeByHeader:
                             #                self.flagEoF = True
                             return 0
                         print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
                         time.sleep(self.delay)
                     return 0
                 def waitDataBlock(self, pointer_location, blocksize=None):
                     currentPointer = pointer_location
                     if blocksize is None:
                         neededSize = self.processingHeaderObj.blockSize  # + self.basicHeaderSize
                     else:
                         neededSize = blocksize
                     for nTries in range(self.nTries):
                         self.fp.close()
                         self.fp = open(self.filename, 'rb')
                         self.fp.seek(currentPointer)
                         self.fileSize = os.path.getsize(self.filename)
                         currentSize = self.fileSize - currentPointer
                         if (currentSize >= neededSize):
                             return 1
                         log.warning(
                             "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
                             self.name
                             )
                         time.sleep(self.delay)
                     return 0
                 def __setNewBlock(self):
                     if self.fp == None:
                         return 0
                     if self.flagIsNewFile:
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if self.realtime:
                         self.flagDiscontinuousBlock = 1
                         if not(self.setNextFile()):
                             return 0
                         else:
                             return 1
                     currentSize = self.fileSize - self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     if (currentSize >= neededSize):
                         self.basicHeaderObj.read(self.fp)
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if self.__waitNewBlock():
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime
                     self.lastUTTime = self.basicHeaderObj.utc
                     self.flagDiscontinuousBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagDiscontinuousBlock = 1
                     return 1
                 def readNextBlock(self):
                     while True:
                         self.__setNewBlock()
                         if not(self.readBlock()):
                             return 0
                         self.getBasicHeader()
                         if not self.isDateTimeInRange(self.dataOut.datatime, self.startDate, self.endDate, self.startTime, self.endTime):
                             print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
                                                                                   self.processingHeaderObj.dataBlocksPerFile,
                                                                                   self.dataOut.datatime.ctime()))
                             continue
                         break
                     if self.verbose:
                         print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
                                                                    self.processingHeaderObj.dataBlocksPerFile,
                                                                    self.dataOut.datatime.ctime()))
                     return 1
                 def readFirstHeader(self):
                     self.basicHeaderObj.read(self.fp)
                     self.systemHeaderObj.read(self.fp)
                     self.radarControllerHeaderObj.read(self.fp)
                     self.processingHeaderObj.read(self.fp)
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
                                                PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
                     else:
                         raise ValueError('Data type was not defined')
                     self.dtype = datatype_str
                     #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
                         self.firstHeaderSize + self.basicHeaderSize * \
                         (self.processingHeaderObj.dataBlocksPerFile - 1)
                     #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
                     #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def verifyFile(self, filename, msgFlag=True):
                     msg = None
                     try:
                         fp = open(filename, 'rb')
                     except IOError:
                         if msgFlag:
                             print("[Reading] File %s can't be opened" % (filename))
                         return False
                     if self.waitDataBlock(0):
                         basicHeaderObj = BasicHeader(LOCALTIME)
                         systemHeaderObj = SystemHeader()
                         radarControllerHeaderObj = RadarControllerHeader()
                         processingHeaderObj = ProcessingHeader()
                         if not(basicHeaderObj.read(fp)):
                             fp.close()
                             return False
                         if not(systemHeaderObj.read(fp)):
                             fp.close()
                             return False
                         if not(radarControllerHeaderObj.read(fp)):
                             fp.close()
                             return False
                         if not(processingHeaderObj.read(fp)):
                             fp.close()
                             return False
                         if not self.online:
                             dt1 = basicHeaderObj.datatime
                             fp.seek(self.fileSize-processingHeaderObj.blockSize-24)
                             if not(basicHeaderObj.read(fp)):
                                 fp.close()
                                 return False
                             dt2 = basicHeaderObj.datatime
                             if not self.isDateTimeInRange(dt1, self.startDate, self.endDate, self.startTime, self.endTime) and not \
                                    self.isDateTimeInRange(dt2, self.startDate, self.endDate, self.startTime, self.endTime):
                                 return False
                     fp.close()
                     return True
                 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
                     path_empty = True
                     dateList = []
                     pathList = []
                     multi_path = path.split(',')
                     if not walk:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             fileList = glob.glob1(single_path, "*" + ext)
                             if not fileList:
                                 continue
                             path_empty = False
                             fileList.sort()
                             for thisFile in fileList:
                                 if not os.path.isfile(os.path.join(single_path, thisFile)):
                                     continue
                                 if not isRadarFile(thisFile):
                                     continue
                                 if not isFileInDateRange(thisFile, startDate, endDate):
                                     continue
                                 thisDate = getDateFromRadarFile(thisFile)
                                 if thisDate in dateList or single_path in pathList:
                                     continue
                                 dateList.append(thisDate)
                                 pathList.append(single_path)
                     else:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             dirList = []
                             for thisPath in os.listdir(single_path):
                                 if not os.path.isdir(os.path.join(single_path, thisPath)):
                                     continue
                                 if not isRadarFolder(thisPath):
                                     continue
                                 if not isFolderInDateRange(thisPath, startDate, endDate):
                                     continue
                                 dirList.append(thisPath)
                             if not dirList:
                                 continue
                             dirList.sort()
                             for thisDir in dirList:
                                 datapath = os.path.join(single_path, thisDir, expLabel)
                                 fileList = glob.glob1(datapath, "*" + ext)
                                 if not fileList:
                                     continue
                                 path_empty = False
                                 thisDate = getDateFromRadarFolder(thisDir)
                                 pathList.append(datapath)
                                 dateList.append(thisDate)
                     dateList.sort()
                     if walk:
                         pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
                     else:
                         pattern_path = multi_path[0]
                     if path_empty:
                         raise schainpy.admin.SchainError("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
                     else:
                         if not dateList:
                             raise schainpy.admin.SchainError("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
                     if include_path:
                         return dateList, pathList
                     return dateList
                 def setup(self, **kwargs):
                     self.set_kwargs(**kwargs)
                     if not self.ext.startswith('.'):
                         self.ext = '.{}'.format(self.ext)
                     if self.server is not None:
                         if 'tcp://' in self.server:
                             address = server
                         else:
                             address = 'ipc:///tmp/%s' % self.server
                         self.server = address
                         self.context = zmq.Context()
                         self.receiver = self.context.socket(zmq.PULL)
                         self.receiver.connect(self.server)
                         time.sleep(0.5)
                         print('[Starting] ReceiverData from {}'.format(self.server))
                     else:
                         self.server = None
                         if self.path == None:
                             raise ValueError("[Reading] The path is not valid")
                         if self.online:
                             log.log("[Reading] Searching files in online mode...", self.name)
                             for nTries in range(self.nTries):
                                 fullpath = self.searchFilesOnLine(self.path, self.startDate,
                                     self.endDate, self.expLabel, self.ext, self.walk,
                                     self.filefmt, self.folderfmt)
                                 try:
                                     fullpath = next(fullpath)
                                 except:
                                     fullpath = None
                                 if fullpath:
                                     break
                                 log.warning(
                                     'Waiting {} sec for a valid file in {}: try {} ...'.format(
                                         self.delay, self.path, nTries + 1),
                                     self.name)
                                 time.sleep(self.delay)
                             if not(fullpath):
                                 raise schainpy.admin.SchainError(
                                     'There isn\'t any valid file in {}'.format(self.path))
                             pathname, filename = os.path.split(fullpath)
                             self.year = int(filename[1:5])
                             self.doy = int(filename[5:8])
                             self.set = int(filename[8:11]) - 1
                         else:
                             log.log("Searching files in {}".format(self.path), self.name)
                             self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
                                 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
                         self.setNextFile()
                     return
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
 . + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                     self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
              #         self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
                 def getFirstHeader(self):
                     raise NotImplementedError
                 def getData(self):
                     raise NotImplementedError
                 def hasNotDataInBuffer(self):
                     raise NotImplementedError
                 def readBlock(self):
                     raise NotImplementedError
                 def isEndProcess(self):
                     return self.flagNoMoreFiles
                 def printReadBlocks(self):
                     print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
                 def printTotalBlocks(self):
                     print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
-                def printNumberOfBlock(self):
-                    'SPAM!'
-             #         if self.flagIsNewBlock:
-             #             print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
-             #                                                       self.processingHeaderObj.dataBlocksPerFile,
-             #                                                       self.dataOut.datatime.ctime())
-                def printInfo(self):
-                    if self.__printInfo == False:
-                        return
-                    self.basicHeaderObj.printInfo()
-                    self.systemHeaderObj.printInfo()
-                    self.radarControllerHeaderObj.printInfo()
-                    self.processingHeaderObj.printInfo()
-                    self.__printInfo = False
                 def run(self, **kwargs):
                     """
                     Arguments:
                         path        :
                         startDate   :
                         endDate     :
                         startTime   :
                         endTime     :
                         set         :
                         expLabel    :
                         ext         :
                         online      :
                         delay       :
                         walk        :
                         getblock    :
                         nTxs        :
                         realtime    :
                         blocksize   :
                         blocktime   :
                         skip        :
                         cursor      :
                         warnings    :
                         server      :
                         verbose     :
                         format      :
                         oneDDict    :
                         twoDDict    :
                         independentParam    :
                     """
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     if self.server is None:
                         self.getData()
                     else:
                         self.getFromServer()
             class JRODataWriter(Reader):
                 """
                 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 setFile = None
                 profilesPerBlock = None
                 blocksPerFile = None
                 nWriteBlocks = 0
                 fileDate = None
                 def __init__(self, dataOut=None):
                     raise NotImplementedError
                 def hasAllDataInBuffer(self):
                     raise NotImplementedError
                 def setBlockDimension(self):
                     raise NotImplementedError
                 def writeBlock(self):
                     raise NotImplementedError
                 def putData(self):
                     raise NotImplementedError
                 def getDtypeWidth(self):
                     dtype_index = get_dtype_index(self.dtype)
                     dtype_width = get_dtype_width(dtype_index)
                     return dtype_width
                 def getProcessFlags(self):
                     processFlags = 0
                     dtype_index = get_dtype_index(self.dtype)
                     procflag_dtype = get_procflag_dtype(dtype_index)
                     processFlags += procflag_dtype
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code is not None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nCohInt > 1:
                         processFlags += PROCFLAG.COHERENT_INTEGRATION
                     if self.dataOut.type == "Spectra":
                         if self.dataOut.nIncohInt > 1:
                             processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                         if self.dataOut.data_dc is not None:
                             processFlags += PROCFLAG.SAVE_CHANNELS_DC
                         if self.dataOut.flagShiftFFT:
                             processFlags += PROCFLAG.SHIFT_FFT_DATA
                     return processFlags
                 def setBasicHeader(self):
                     self.basicHeaderObj.size = self.basicHeaderSize  # bytes
                     self.basicHeaderObj.version = self.versionFile
                     self.basicHeaderObj.dataBlock = self.nTotalBlocks
                     utc = numpy.floor(self.dataOut.utctime)
                     milisecond = (self.dataOut.utctime - utc) * 1000.0
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = milisecond
                     self.basicHeaderObj.timeZone = self.dataOut.timeZone
                     self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                     self.basicHeaderObj.errorCount = self.dataOut.errorCount
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj self.
                     Return:
                         None
                     """
                     raise NotImplementedError
                 def __writeFirstHeader(self):
                     """
                     Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                     Affected:
                         __dataType
                     Return:
                         None
                     """
             #        CALCULAR PARAMETROS
                     sizeLongHeader = self.systemHeaderObj.size + \
                         self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                     self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                     self.basicHeaderObj.write(self.fp)
                     self.systemHeaderObj.write(self.fp)
                     self.radarControllerHeaderObj.write(self.fp)
                     self.processingHeaderObj.write(self.fp)
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                     """
                     if self.fp == None:
                         self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                         self.basicHeaderObj.write(self.fp)
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     """
                     Selecciona el bloque siguiente de datos y los escribe en un file
                     Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                     """
                     if not(self.__setNewBlock()):
                         return 0
                     self.writeBlock()
                     print("[Writing] Block No. %d/%d" % (self.blockIndex,
                                                          self.processingHeaderObj.dataBlocksPerFile))
                     return 1
                 def setNextFile(self):
                     """Determina el siguiente file que sera escrito
                     Affected:
                         self.filename
                         self.subfolder
                         self.fp
                         self.setFile
                         self.flagIsNewFile
                     Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                     """
                     ext = self.ext
                     path = self.path
                     if self.fp != None:
                         self.fp.close()
                     if not os.path.exists(path):
                         os.mkdir(path)
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
                     fullpath = os.path.join(path, subfolder)
                     setFile = self.setFile
                     if not(os.path.exists(fullpath)):
                         os.mkdir(fullpath)
                         setFile = -1  # inicializo mi contador de seteo
                     else:
                         filesList = os.listdir(fullpath)
                         if len(filesList) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 # inicializo mi contador de seteo al seteo del ultimo file
                                 setFile = int(filen[8:11])
                             else:
                                 setFile = -1
                         else:
                             setFile = -1  # inicializo mi contador de seteo
                     setFile += 1
                     # If this is a new day it resets some values
                     if self.dataOut.datatime.date() > self.fileDate:
                         setFile = 0
                         self.nTotalBlocks = 0
                     filen = '{}{:04d}{:03d}{:03d}{}'.format(
                             self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
                     filename = os.path.join(path, subfolder, filen)
                     fp = open(filename, 'wb')
                     self.blockIndex = 0
                     self.filename = filename
                     self.subfolder = subfolder
                     self.fp = fp
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     self.fileDate = self.dataOut.datatime.date()
                     self.setFirstHeader()
                     print('[Writing] Opening file: %s' % self.filename)
                     self.__writeFirstHeader()
                     return 1
                 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
                     """
                     Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                     Inputs:
                         path                :    directory where data will be saved
                         profilesPerBlock    :    number of profiles per block
                         set                 :    initial file set
                         datatype            :    An integer number that defines data type:
 : int8  (1 byte)
 : int16 (2 bytes)
 : int32 (4 bytes)
 : int64 (8 bytes)
 : float32 (4 bytes)
 : double64 (8 bytes)
                     Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                     """
                     if ext == None:
                         ext = self.ext
                     self.ext = ext.lower()
                     self.path = path
                     if set is None:
                         self.setFile = -1
                     else:
                         self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile
                     self.profilesPerBlock = profilesPerBlock
                     self.dataOut = dataOut
                     self.fileDate = self.dataOut.datatime.date()
                     self.dtype = self.dataOut.dtype
                     if datatype is not None:
                         self.dtype = get_numpy_dtype(datatype)
                     if not(self.setNextFile()):
                         print("[Writing] There isn't a next file")
                         return 0
                     self.setBlockDimension()
                     return 1
                 def run(self, dataOut, path, blocksPerFile=100, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
                                    set=set, ext=ext, datatype=datatype, **kwargs)
                         self.isConfig = True
                     self.dataOut = dataOut
                     self.putData()
                     return self.dataOut
+            class printInfo(Operation):
+                def __init__(self):
+                    Operation.__init__(self)
+                    self.__printInfo = True
+                def run(self, dataOut, headers = ['systemHeaderObj', 'radarControllerHeaderObj', 'processingHeaderObj']):
+                    if self.__printInfo == False:
+                        return dataOut
+                    for header in headers:
+                        if hasattr(dataOut, header):
+                            obj = getattr(dataOut, header)
+                            if hasattr(obj, 'printInfo'):
+                                obj.printInfo()
+                            else:
+                                print(obj)
+                        else:
+                            log.warning('Header {} Not found in object'.format(header))
+                    self.__printInfo = False
+                    return dataOut
  No newline at end of file

schainpy/model/io/jroIO_digitalRF.py +2 -2

             '''
             Created on Jul 3, 2014
             @author: roj-idl71
             '''
             # SUBCHANNELS EN VEZ DE CHANNELS
             # BENCHMARKS -> PROBLEMAS CON ARCHIVOS GRANDES -> INCONSTANTE EN EL TIEMPO
             # ACTUALIZACION DE VERSION
             # HEADERS
             # MODULO DE ESCRITURA
             # METADATA
             import os
             import time
             import datetime
             import numpy
             import timeit
             from fractions import Fraction
             from time import time
             from time import sleep
             import schainpy.admin
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             import pickle
             try:
                 import digital_rf
             except:
                 pass
-            @MPDecorator
             class DigitalRFReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self)
                     self.dataOut                  = Voltage()
                     self.__printInfo              = True
                     self.__flagDiscontinuousBlock = False
                     self.__bufferIndex = 9999999
                     self.__codeType    = 0
                     self.__ippKm       = None
                     self.__nCode       = None
                     self.__nBaud       = None
                     self.__code        = None
                     self.dtype         = None
                     self.oldAverage    = None
                     self.path          = None
                 def close(self):
                     print('Average of writing to digital rf format is ', self.oldAverage * 1000)
                     return
                 def __getCurrentSecond(self):
                     return self.__thisUnixSample / self.__sample_rate
                 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
                 def __setFileHeader(self):
                     '''
                     In this method will be initialized every parameter of dataOut object (header, no data)
                     '''
                     ippSeconds = 1.0 * self.__nSamples / self.__sample_rate
                     nProfiles = 1.0 / ippSeconds  # Number of profiles in one second
                     try:
                         self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
                             self.__radarControllerHeader)
                     except:
                         self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
                             txA=0,
                             txB=0,
                             nWindows=1,
                             nHeights=self.__nSamples,
                             firstHeight=self.__firstHeigth,
                             deltaHeight=self.__deltaHeigth,
                             codeType=self.__codeType,
                             nCode=self.__nCode, nBaud=self.__nBaud,
                             code=self.__code)
                     try:
                         self.dataOut.systemHeaderObj = SystemHeader(self.__systemHeader)
                     except:
                         self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                     nProfiles=nProfiles,
                                                                     nChannels=len(
                                                                         self.__channelList),
                                                                     adcResolution=14)
                     self.dataOut.type  = "Voltage"
                     self.dataOut.data  = None
                     self.dataOut.dtype = self.dtype
                     # self.dataOut.nChannels = 0
                     # self.dataOut.nHeights = 0
                     self.dataOut.nProfiles   = int(nProfiles)
                     self.dataOut.heightList  = self.__firstHeigth + \
                         numpy.arange(self.__nSamples, dtype=numpy.float) * \
                         self.__deltaHeigth
                     self.dataOut.channelList = list(range(self.__num_subchannels))
                     self.dataOut.blocksize   = self.dataOut.getNChannels() * self.dataOut.getNHeights()
                     # self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData      = True
                     self.dataOut.flagDataAsBlock = False
                     # Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime     = None
                     # timezone like jroheader, difference in minutes between UTC and localtime
                     self.dataOut.timeZone    = self.__timezone / 60
                     self.dataOut.dstFlag     = 0
                     self.dataOut.errorCount  = 0
                     try:
                         self.dataOut.nCohInt = self.fixed_metadata_dict.get(
                             'nCohInt', self.nCohInt)
                         # asumo que la data esta decodificada
                         self.dataOut.flagDecodeData = self.fixed_metadata_dict.get(
                             'flagDecodeData', self.flagDecodeData)
                         # asumo que la data esta sin flip
                         self.dataOut.flagDeflipData = self.fixed_metadata_dict['flagDeflipData']
                         self.dataOut.flagShiftFFT   = self.fixed_metadata_dict['flagShiftFFT']
                         self.dataOut.useLocalTime   = self.fixed_metadata_dict['useLocalTime']
                     except:
                         pass
                     self.dataOut.ippSeconds = ippSeconds
                     # Time interval between profiles
                     # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency  = self.__frequency
                     self.dataOut.realtime   = self.__online
                 def findDatafiles(self, path, startDate=None, endDate=None):
                     if not os.path.isdir(path):
                         return []
                     try:
                         digitalReadObj = digital_rf.DigitalRFReader(
                             path, load_all_metadata=True)
                     except:
                         digitalReadObj = digital_rf.DigitalRFReader(path)
                     channelNameList    = digitalReadObj.get_channels()
                     if not channelNameList:
                         return []
                     metadata_dict      = digitalReadObj.get_rf_file_metadata(channelNameList[0])
                     sample_rate        = metadata_dict['sample_rate'][0]
                     this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
                     try:
                         timezone       = this_metadata_file['timezone'].value
                     except:
                         timezone       = 0
                     startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(
                         channelNameList[0]) / sample_rate - timezone
                     startDatetime      = datetime.datetime.utcfromtimestamp(startUTCSecond)
                     endDatatime        = datetime.datetime.utcfromtimestamp(endUTCSecond)
                     if not startDate:
                         startDate      = startDatetime.date()
                     if not endDate:
                         endDate        = endDatatime.date()
                     dateList           = []
                     thisDatetime       = startDatetime
                     while(thisDatetime <= endDatatime):
                         thisDate        = thisDatetime.date()
                         if thisDate     < startDate:
                             continue
                         if thisDate     > endDate:
                             break
                         dateList.append(thisDate)
                         thisDatetime += datetime.timedelta(1)
                     return dateList
                 def setup(self, path=None,
                           startDate=None,
                           endDate=None,
                           startTime=datetime.time(0, 0, 0),
                           endTime=datetime.time(23, 59, 59),
                           channelList=None,
                           nSamples=None,
                           online=False,
                           delay=60,
                           buffer_size=1024,
                           ippKm=None,
                           nCohInt=1,
                           nCode=1,
                           nBaud=1,
                           flagDecodeData=False,
                           code=numpy.ones((1, 1), dtype=numpy.int),
                           **kwargs):
                     '''
                     In this method we should set all initial parameters.
                     Inputs:
                         path
                         startDate
                         endDate
                         startTime
                         endTime
                         set
                         expLabel
                         ext
                         online
                         delay
                     '''
                     self.path           = path
                     self.nCohInt        = nCohInt
                     self.flagDecodeData = flagDecodeData
                     self.i              = 0
                     if not os.path.isdir(path):
                         raise ValueError("[Reading] Directory %s does not exist" % path)
                     try:
                         self.digitalReadObj = digital_rf.DigitalRFReader(
                             path, load_all_metadata=True)
                     except:
                         self.digitalReadObj = digital_rf.DigitalRFReader(path)
                     channelNameList         = self.digitalReadObj.get_channels()
                     if not channelNameList:
                         raise ValueError("[Reading] Directory %s does not have any files" % path)
                     if not channelList:
                         channelList = list(range(len(channelNameList)))
                     ##########  Reading metadata ######################
                     top_properties           = self.digitalReadObj.get_properties(
                         channelNameList[channelList[0]])
                     self.__num_subchannels   = top_properties['num_subchannels']
                     self.__sample_rate       = 1.0 * \
                         top_properties['sample_rate_numerator'] / \
                         top_properties['sample_rate_denominator']
                     # self.__samples_per_file = top_properties['samples_per_file'][0]
                     self.__deltaHeigth       = 1e6 * 0.15 / self.__sample_rate  # why 0.15?
                     this_metadata_file       = self.digitalReadObj.get_digital_metadata(
                         channelNameList[channelList[0]])
                     metadata_bounds          = this_metadata_file.get_bounds()
                     self.fixed_metadata_dict = this_metadata_file.read(
                         metadata_bounds[0])[metadata_bounds[0]]  # GET FIRST HEADER
                     try:
                         self.__processingHeader      = self.fixed_metadata_dict['processingHeader']
                         self.__radarControllerHeader = self.fixed_metadata_dict['radarControllerHeader']
                         self.__systemHeader          = self.fixed_metadata_dict['systemHeader']
                         self.dtype                   = pickle.loads(self.fixed_metadata_dict['dtype'])
                     except:
                         pass
                     self.__frequency = None
                     self.__frequency = self.fixed_metadata_dict.get('frequency', 1)
                     self.__timezone = self.fixed_metadata_dict.get('timezone', 18000)
                     try:
                         nSamples = self.fixed_metadata_dict['nSamples']
                     except:
                         nSamples = None
                     self.__firstHeigth = 0
                     try:
                         codeType       = self.__radarControllerHeader['codeType']
                     except:
                         codeType       = 0
                     try:
                         if codeType:
                             nCode = self.__radarControllerHeader['nCode']
                             nBaud = self.__radarControllerHeader['nBaud']
                             code  = self.__radarControllerHeader['code']
                     except:
                         pass
                     if not ippKm:
                         try:
                             # seconds to km
                             ippKm = self.__radarControllerHeader['ipp']
                         except:
                             ippKm = None
                     ####################################################
                     self.__ippKm   = ippKm
                     startUTCSecond = None
                     endUTCSecond   = None
                     if startDate:
                         startDatetime  = datetime.datetime.combine(startDate, startTime)
                         startUTCSecond = (
                             startDatetime - datetime.datetime(1970, 1, 1)).total_seconds() + self.__timezone
                     if endDate:
                         endDatetime   = datetime.datetime.combine(endDate, endTime)
                         endUTCSecond  = (endDatetime - datetime.datetime(1970,
 , 1)).total_seconds() + self.__timezone
                     start_index, end_index = self.digitalReadObj.get_bounds(
                         channelNameList[channelList[0]])
                     if not startUTCSecond:
                         startUTCSecond = start_index / self.__sample_rate
                     if start_index     > startUTCSecond * self.__sample_rate:
                         startUTCSecond = start_index / self.__sample_rate
                     if not endUTCSecond:
                         endUTCSecond   = end_index / self.__sample_rate
                     if end_index       < endUTCSecond * self.__sample_rate:
                         endUTCSecond   = end_index / self.__sample_rate
                     if not nSamples:
                         if not ippKm:
                             raise ValueError("[Reading] nSamples or ippKm should be defined")
                         nSamples            = int(ippKm / (1e6 * 0.15 / self.__sample_rate))
                     channelBoundList        = []
                     channelNameListFiltered = []
                     for thisIndexChannel in channelList:
                         thisChannelName        = channelNameList[thisIndexChannel]
                         start_index, end_index = self.digitalReadObj.get_bounds(
                             thisChannelName)
                         channelBoundList.append((start_index, end_index))
                         channelNameListFiltered.append(thisChannelName)
                     self.profileIndex = 0
                     self.i            = 0
                     self.__delay      = delay
                     self.__codeType   = codeType
                     self.__nCode      = nCode
                     self.__nBaud      = nBaud
                     self.__code       = code
                     self.__datapath         = path
                     self.__online           = online
                     self.__channelList      = channelList
                     self.__channelNameList  = channelNameListFiltered
                     self.__channelBoundList = channelBoundList
                     self.__nSamples         = nSamples
                     self.__samples_to_read  = int(nSamples)  # FIJO: AHORA 40
                     self.__nChannels        = len(self.__channelList)
                     self.__startUTCSecond   = startUTCSecond
                     self.__endUTCSecond     = endUTCSecond
                     self.__timeInterval     = 1.0 * self.__samples_to_read / \
                         self.__sample_rate  # Time interval
                     if online:
                         # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
                         startUTCSecond = numpy.floor(endUTCSecond)
                     # por que en el otro metodo lo primero q se hace es sumar samplestoread
                     self.__thisUnixSample = int(startUTCSecond * self.__sample_rate) - self.__samples_to_read
                     self.__data_buffer    = numpy.zeros(
                         (self.__num_subchannels, self.__samples_to_read), dtype=numpy.complex)
                     self.__setFileHeader()
                     self.isConfig = True
                     print("[Reading] Digital RF Data was found from %s to %s " % (
                         datetime.datetime.utcfromtimestamp(
                             self.__startUTCSecond - self.__timezone),
                         datetime.datetime.utcfromtimestamp(
                             self.__endUTCSecond - self.__timezone)
                     ))
                     print("[Reading] Starting process from %s to %s" % (datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
                                                                         datetime.datetime.utcfromtimestamp(
                         endUTCSecond - self.__timezone)
                     ))
                     self.oldAverage    = None
                     self.count         = 0
                     self.executionTime = 0
                 def __reload(self):
                     #         print
                     #         print "%s not in range [%s, %s]" %(
                     #                                           datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                     #                                           datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
                     #                                           datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
                     #                                           )
                     print("[Reading] reloading metadata ...")
                     try:
                         self.digitalReadObj.reload(complete_update=True)
                     except:
                         self.digitalReadObj = digital_rf.DigitalRFReader(self.path)
                     start_index, end_index  = self.digitalReadObj.get_bounds(
                         self.__channelNameList[self.__channelList[0]])
                     if start_index          > self.__startUTCSecond * self.__sample_rate:
                         self.__startUTCSecond = 1.0 * start_index / self.__sample_rate
                     if end_index            > self.__endUTCSecond * self.__sample_rate:
                         self.__endUTCSecond = 1.0 * end_index / self.__sample_rate
                         print()
                         print("[Reading] New timerange found [%s, %s] " % (
                             datetime.datetime.utcfromtimestamp(
                                 self.__startUTCSecond - self.__timezone),
                             datetime.datetime.utcfromtimestamp(
                                 self.__endUTCSecond - self.__timezone)
                         ))
                         return True
                     return False
                 def timeit(self, toExecute):
                     t0                  = time.time()
                     toExecute()
                     self.executionTime  = time.time() - t0
                     if self.oldAverage is None:
                         self.oldAverage = self.executionTime
                     self.oldAverage     = (self.executionTime + self.count *
                                        self.oldAverage) / (self.count + 1.0)
                     self.count          = self.count + 1.0
                     return
                 def __readNextBlock(self, seconds=30, volt_scale=1):
                     '''
                     '''
                     # Set the next data
                     self.__flagDiscontinuousBlock = False
                     self.__thisUnixSample        += self.__samples_to_read
                     if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
                         print ("[Reading] There are no more data into selected time-range")
                         if self.__online:
                             sleep(3)
                             self.__reload()
                         else:
                             return False
                         if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
                             return False
                             self.__thisUnixSample -= self.__samples_to_read
                     indexChannel = 0
                     dataOk = False
                     for thisChannelName in self.__channelNameList:  # TODO VARIOS CHANNELS?
                         for indexSubchannel in range(self.__num_subchannels):
                             try:
                                 t0     = time()
                                 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
                                                                               self.__samples_to_read,
                                                                               thisChannelName, sub_channel=indexSubchannel)
                                 self.executionTime  = time() - t0
                                 if self.oldAverage is None:
                                     self.oldAverage = self.executionTime
                                 self.oldAverage     = (
                                     self.executionTime + self.count * self.oldAverage) / (self.count + 1.0)
                                 self.count = self.count + 1.0
                             except IOError as e:
                                 # read next profile
                                 self.__flagDiscontinuousBlock = True
                                 print("[Reading] %s" % datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e)
                                 break
                             if result.shape[0] != self.__samples_to_read:
                                 self.__flagDiscontinuousBlock = True
                                 print("[Reading] %s: Too few samples were found, just %d/%d  samples" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                                                          result.shape[0],
                                                                                                          self.__samples_to_read))
                                 break
                             self.__data_buffer[indexSubchannel, :] = result * volt_scale
                             indexChannel+=1
                             dataOk       = True
                     self.__utctime       = self.__thisUnixSample / self.__sample_rate
                     if not dataOk:
                         return False
                     print("[Reading] %s: %d samples <> %f sec" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                   self.__samples_to_read,
                                                                   self.__timeInterval))
                     self.__bufferIndex   = 0
                     return True
                 def __isBufferEmpty(self):
                     return self.__bufferIndex > self.__samples_to_read - self.__nSamples  # 40960 - 40
                 def getData(self, seconds=30, nTries=5):
                     '''
                         This method gets the data from files and put the data into the dataOut object
                         In addition, increase el the buffer counter in one.
                         Return:
                             data    :    retorna un perfil de voltages (alturas * canales) copiados desde el
                                         buffer. Si no hay mas archivos a leer retorna None.
                         Affected:
                             self.dataOut
                             self.profileIndex
                             self.flagDiscontinuousBlock
                             self.flagIsNewBlock
                     '''
                     #print("getdata")
                     err_counter = 0
                     self.dataOut.flagNoData = True
                     if self.__isBufferEmpty():
                         #print("hi")
                         self.__flagDiscontinuousBlock = False
                         while True:
                             #print ("q ha pasado")
                             if self.__readNextBlock():
                                 break
                             if self.__thisUnixSample > self.__endUTCSecond * self.__sample_rate:
                                 raise schainpy.admin.SchainError('Error')
                                 return
                             if self.__flagDiscontinuousBlock:
                                 raise schainpy.admin.SchainError('discontinuous block found')
                                 return
                             if not self.__online:
                                 raise schainpy.admin.SchainError('Online?')
                                 return
                             err_counter += 1
                             if err_counter > nTries:
                                 raise schainpy.admin.SchainError('Max retrys reach')
                                 return
                             print('[Reading] waiting %d seconds to read a new block' % seconds)
                             time.sleep(seconds)
                     self.dataOut.data                   = self.__data_buffer[:, self.__bufferIndex:self.__bufferIndex + self.__nSamples]
                     self.dataOut.utctime                = ( self.__thisUnixSample + self.__bufferIndex) / self.__sample_rate
                     self.dataOut.flagNoData             = False
                     self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
                     self.dataOut.profileIndex           = self.profileIndex
                     self.__bufferIndex += self.__nSamples
                     self.profileIndex  += 1
                     if self.profileIndex == self.dataOut.nProfiles:
                         self.profileIndex = 0
                     return True
                 def printInfo(self):
                     '''
                     '''
                     if self.__printInfo == False:
                         return
                     # self.systemHeaderObj.printInfo()
                     # self.radarControllerHeaderObj.printInfo()
                     self.__printInfo = False
                 def printNumberOfBlock(self):
                     '''
                     '''
                     return
                     # print self.profileIndex
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                     #self.i = self.i+1
                     self.getData(seconds=self.__delay)
                     return
+            @MPDecorator
             class DigitalRFWriter(Operation):
                 '''
                 classdocs
                 '''
                 def __init__(self, **kwargs):
                     '''
                     Constructor
                     '''
                     Operation.__init__(self, **kwargs)
                     self.metadata_dict = {}
                     self.dataOut       = None
                     self.dtype         = None
                     self.oldAverage    = 0
                 def setHeader(self):
                     self.metadata_dict['frequency']      = self.dataOut.frequency
                     self.metadata_dict['timezone']       = self.dataOut.timeZone
                     self.metadata_dict['dtype']          = pickle.dumps(self.dataOut.dtype)
                     self.metadata_dict['nProfiles']      = self.dataOut.nProfiles
                     self.metadata_dict['heightList']     = self.dataOut.heightList
                     self.metadata_dict['channelList']    = self.dataOut.channelList
                     self.metadata_dict['flagDecodeData'] = self.dataOut.flagDecodeData
                     self.metadata_dict['flagDeflipData'] = self.dataOut.flagDeflipData
                     self.metadata_dict['flagShiftFFT']   = self.dataOut.flagShiftFFT
                     self.metadata_dict['useLocalTime']   = self.dataOut.useLocalTime
                     self.metadata_dict['nCohInt']        = self.dataOut.nCohInt
                     self.metadata_dict['type']           = self.dataOut.type
                     self.metadata_dict['flagDataAsBlock']= getattr(
                         self.dataOut, 'flagDataAsBlock', None)  # chequear
                 def setup(self, dataOut, path, frequency, fileCadence, dirCadence, metadataCadence, set=0, metadataFile='metadata', ext='.h5'):
                     '''
                     In this method we should set all initial parameters.
                     Input:
                         dataOut: Input data will also be outputa data
                     '''
                     self.setHeader()
                     self.__ippSeconds  = dataOut.ippSeconds
                     self.__deltaH      = dataOut.getDeltaH()
                     self.__sample_rate = 1e6 * 0.15 / self.__deltaH
                     self.__dtype       = dataOut.dtype
                     if len(dataOut.dtype) == 2:
                         self.__dtype = dataOut.dtype[0]
                     self.__nSamples  = dataOut.systemHeaderObj.nSamples
                     self.__nProfiles = dataOut.nProfiles
                     if self.dataOut.type != 'Voltage':
                         raise 'Digital RF cannot be used with this data type'
                         self.arr_data = numpy.ones((1, dataOut.nFFTPoints * len(
                             self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
                     else:
                         self.arr_data = numpy.ones((self.__nSamples, len(
                             self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
                     file_cadence_millisecs  = 1000
                     sample_rate_fraction    = Fraction(self.__sample_rate).limit_denominator()
                     sample_rate_numerator   = int(sample_rate_fraction.numerator)
                     sample_rate_denominator = int(sample_rate_fraction.denominator)
                     start_global_index      = dataOut.utctime * self.__sample_rate
                     uuid              = 'prueba'
                     compression_level = 0
                     checksum          = False
                     is_complex        = True
                     num_subchannels   = len(dataOut.channelList)
                     is_continuous     = True
                     marching_periods  = False
                     self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, dirCadence,
                                                                       fileCadence, start_global_index,
                                                                       sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum,
                                                                       is_complex, num_subchannels, is_continuous, marching_periods)
                     metadata_dir         = os.path.join(path, 'metadata')
                     os.system('mkdir %s' % (metadata_dir))
                     self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, dirCadence, 1,  # 236, file_cadence_millisecs / 1000
                                                                                     sample_rate_numerator, sample_rate_denominator,
                                                                                     metadataFile)
                     self.isConfig      = True
                     self.currentSample = 0
                     self.oldAverage    = 0
                     self.count         = 0
                     return
                 def writeMetadata(self):
                     start_idx                                   = self.__sample_rate * self.dataOut.utctime
                     self.metadata_dict['processingHeader']      = self.dataOut.processingHeaderObj.getAsDict(
                     )
                     self.metadata_dict['radarControllerHeader'] = self.dataOut.radarControllerHeaderObj.getAsDict(
                     )
                     self.metadata_dict['systemHeader']          = self.dataOut.systemHeaderObj.getAsDict(
                     )
                     self.digitalMetadataWriteObj.write(start_idx, self.metadata_dict)
                     return
                 def timeit(self, toExecute):
                     t0 = time()
                     toExecute()
                     self.executionTime  = time() - t0
                     if self.oldAverage is None:
                         self.oldAverage = self.executionTime
                     self.oldAverage     = (self.executionTime + self.count *
                                        self.oldAverage) / (self.count + 1.0)
                     self.count          = self.count + 1.0
                     return
                 def writeData(self):
                     if self.dataOut.type != 'Voltage':
                         raise 'Digital RF cannot be used with this data type'
                         for channel in self.dataOut.channelList:
                             for i in range(self.dataOut.nFFTPoints):
                                 self.arr_data[1][channel * self.dataOut.nFFTPoints +
                                                  i]['r'] = self.dataOut.data[channel][i].real
                                 self.arr_data[1][channel * self.dataOut.nFFTPoints +
                                                  i]['i'] = self.dataOut.data[channel][i].imag
                     else:
                         for i in range(self.dataOut.systemHeaderObj.nSamples):
                             for channel in self.dataOut.channelList:
                                 self.arr_data[i][channel]['r'] = self.dataOut.data[channel][i].real
                                 self.arr_data[i][channel]['i'] = self.dataOut.data[channel][i].imag
                     def f(): return self.digitalWriteObj.rf_write(self.arr_data)
                     self.timeit(f)
                     return
                 def run(self, dataOut, frequency=49.92e6, path=None, fileCadence=1000, dirCadence=36000, metadataCadence=1, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     Inputs:
                         dataOut: object with the data
                     '''
                     # print dataOut.__dict__
                     self.dataOut = dataOut
                     if not self.isConfig:
                         self.setup(dataOut, path, frequency, fileCadence,
                                    dirCadence, metadataCadence, **kwargs)
                         self.writeMetadata()
                     self.writeData()
                     ## self.currentSample += 1
                     # if self.dataOut.flagDataAsBlock or self.currentSample == 1:
                     # self.writeMetadata()
                     ## if self.currentSample == self.__nProfiles: self.currentSample = 0
                     return dataOut# en la version 2.7 no aparece este return
                 def close(self):
                     print('[Writing] - Closing files ')
                     print('Average of writing to digital rf format is ', self.oldAverage * 1000)
                     try:
                         self.digitalWriteObj.close()
                     except:
                         pass

schainpy/model/io/jroIO_heispectra.py +2 -1

             '''
             Created on Jul 3, 2014
             @author: roj-idl71
             '''
             import os, sys
             import time, datetime
             import numpy
             import fnmatch
             import glob
             from time import sleep
             try:
                 import pyfits
             except ImportError as e:
                 pass
             from xml.etree.ElementTree import ElementTree
             from .jroIO_base import isRadarFolder, isNumber
             from schainpy.model.data.jrodata import Fits
             from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit, MPDecorator
             from schainpy.utils import log
             class PyFits(object):
                 name=None
                 format=None
                 array =None
                 data =None
                 thdulist=None
                 prihdr=None
                 hdu=None
                 def __init__(self):
                     pass
                 def setColF(self,name,format,array):
                     self.name=name
                     self.format=format
                     self.array=array
                     a1=numpy.array([self.array],dtype=numpy.float32)
                     self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
                     return self.col1
             #    def setColP(self,name,format,data):
             #        self.name=name
             #        self.format=format
             #        self.data=data
             #        a2=numpy.array([self.data],dtype=numpy.float32)
             #        self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
             #        return self.col2
                 def writeData(self,name,format,data):
                     self.name=name
                     self.format=format
                     self.data=data
                     a2=numpy.array([self.data],dtype=numpy.float32)
                     self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
                     return self.col2
                 def cFImage(self,idblock,year,month,day,hour,minute,second):
                     self.hdu= pyfits.PrimaryHDU(idblock)
                     self.hdu.header.set("Year",year)
                     self.hdu.header.set("Month",month)
                     self.hdu.header.set("Day",day)
                     self.hdu.header.set("Hour",hour)
                     self.hdu.header.set("Minute",minute)
                     self.hdu.header.set("Second",second)
                     return self.hdu
                 def Ctable(self,colList):
                     self.cols=pyfits.ColDefs(colList)
                     self.tbhdu = pyfits.new_table(self.cols)
                     return self.tbhdu
                 def CFile(self,hdu,tbhdu):
                     self.thdulist=pyfits.HDUList([hdu,tbhdu])
                 def wFile(self,filename):
                     if os.path.isfile(filename):
                         os.remove(filename)
                     self.thdulist.writeto(filename)
             class ParameterConf:
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.name = ''
                     self.value = ''
                 def readXml(self, parmElement):
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                 def getElementName(self):
                     return self.ELEMENTNAME
             class Metadata(object):
                 def __init__(self, filename):
                     self.parmConfObjList = []
                     self.readXml(filename)
                 def readXml(self, filename):
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     self.projectElement = ElementTree().parse(filename)
                     self.project = self.projectElement.tag
                     parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         self.parmConfObjList.append(parmConfObj)
+            @MPDecorator
             class FitsWriter(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.isConfig = False
                     self.dataBlocksPerFile = None
                     self.blockIndex = 0
                     self.flagIsNewFile = 1
                     self.fitsObj = None
                     self.optchar = 'P'
                     self.ext = '.fits'
                     self.setFile = 0
                 def setFitsHeader(self, dataOut, metadatafile=None):
                     header_data = pyfits.PrimaryHDU()
                     header_data.header['EXPNAME'] = "RADAR DATA"
                     header_data.header['DATATYPE'] = "SPECTRA"
                     header_data.header['COMMENT'] = ""
                     if metadatafile:
                         metadata4fits = Metadata(metadatafile)
                         for parameter in metadata4fits.parmConfObjList:
                             parm_name = parameter.name
                             parm_value = parameter.value
                             header_data.header[parm_name] = parm_value
                     header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
                     header_data.header['CHANNELLIST'] = str(dataOut.channelList)
                     header_data.header['NCHANNELS'] = dataOut.nChannels
                     #header_data.header['HEIGHTS'] = dataOut.heightList
                     header_data.header['NHEIGHTS'] = dataOut.nHeights
                     header_data.header['IPPSECONDS'] = dataOut.ippSeconds
                     header_data.header['NCOHINT'] = dataOut.nCohInt
                     header_data.header['NINCOHINT'] = dataOut.nIncohInt
                     header_data.header['TIMEZONE'] = dataOut.timeZone
                     header_data.header['NBLOCK'] = self.blockIndex
                     header_data.writeto(self.filename)
                     self.addExtension(dataOut.heightList,'HEIGHTLIST')
                 def setup(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None):
                     self.path = path
                     self.dataOut = dataOut
                     self.metadatafile = metadatafile
                     self.dataBlocksPerFile = dataBlocksPerFile
                 def open(self):
                     self.fitsObj = pyfits.open(self.filename, mode='update')
                 def addExtension(self, data, tagname):
                     self.open()
                     extension = pyfits.ImageHDU(data=data, name=tagname)
                     #extension.header['TAG'] = tagname
                     self.fitsObj.append(extension)
                     self.write()
                 def addData(self, data):
                     self.open()
                     extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])
                     extension.header['UTCTIME'] = self.dataOut.utctime
                     self.fitsObj.append(extension)
                     self.blockIndex += 1
                     self.fitsObj[0].header['NBLOCK'] = self.blockIndex
                     self.write()
                 def write(self):
                     self.fitsObj.flush(verbose=True)
                     self.fitsObj.close()
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         self.setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             if isNumber( filen[8:11] ):
                                 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 self.setFile = -1
                         else:
                             self.setFile = -1 #inicializo mi contador de seteo
                     setFile = self.setFile
                     setFile += 1
                     thisFile = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, thisFile )
                     self.blockIndex = 0
                     self.filename = filename
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     print('Writing the file: %s'%self.filename)
                     self.setFitsHeader(self.dataOut, self.metadatafile)
                     return 1
                 def writeBlock(self):
                     self.addData(self.dataOut.data_spc)
                     self.flagIsNewFile = 0
                 def __setNewBlock(self):
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.dataBlocksPerFile:
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     return 1
                 def putData(self):
                     if self.flagIsNewFile:
                         self.setNextFile()
                     self.writeNextBlock()
                 def run(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, path, dataBlocksPerFile=dataBlocksPerFile, metadatafile=metadatafile, **kwargs)
                         self.isConfig = True
                     self.putData()
-            @MPDecorator
             class FitsReader(ProcessingUnit):
             #     __TIMEZONE = time.timezone
                 expName = None
                 datetimestr = None
                 utc = None
                 nChannels = None
                 nSamples = None
                 dataBlocksPerFile = None
                 comments = None
                 lastUTTime = None
                 header_dict = None
                 data = None
                 data_header_dict = None
                 def __init__(self):#, **kwargs):
                     ProcessingUnit.__init__(self)#, **kwargs)
                     self.isConfig = False
                     self.ext = '.fits'
                     self.setFile = 0
                     self.flagNoMoreFiles = 0
                     self.flagIsNewFile = 1
                     self.flagDiscontinuousBlock = None
                     self.fileIndex = None
                     self.filename = None
                     self.fileSize = None
                     self.fitsObj = None
                     self.timeZone = None
                     self.nReadBlocks = 0
                     self.nTotalBlocks = 0
                     self.dataOut = self.createObjByDefault()
                     self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
                     self.blockIndex = 1
                 def createObjByDefault(self):
                     dataObj = Fits()
                     return dataObj
                 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
                     try:
                         fitsObj = pyfits.open(filename,'readonly')
                     except:
                         print("File %s can't be opened" %(filename))
                         return None
                     header = fitsObj[0].header
                     struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
                     utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
                     ltc = utc
                     if useLocalTime:
                         ltc -= time.timezone
                     thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
                     thisTime = thisDatetime.time()
                     if not ((startTime <= thisTime) and (endTime > thisTime)):
                         return None
                     return thisDatetime
                 def __setNextFileOnline(self):
                     raise NotImplementedError
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print("No more Files")
                             return 0
                         filename = self.filenameList[idFile]
             #            if not(self.__verifyFile(filename)):
             #                continue
                         fileSize = os.path.getsize(filename)
                         fitsObj = pyfits.open(filename,'readonly')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fitsObj = fitsObj
                     self.blockIndex = 0
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def __setValuesFromHeader(self):
                     self.dataOut.header = self.header_dict
                     self.dataOut.expName = self.expName
                     self.dataOut.timeZone = self.timeZone
                     self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
                     self.dataOut.comments = self.comments
             #         self.dataOut.timeInterval = self.timeInterval
                     self.dataOut.channelList = self.channelList
                     self.dataOut.heightList = self.heightList
                     self.dataOut.nCohInt = self.nCohInt
                     self.dataOut.nIncohInt = self.nIncohInt
                     self.dataOut.ipp_sec = self.ippSeconds
                 def readHeader(self):
                     headerObj = self.fitsObj[0]
                     self.header_dict = headerObj.header
                     if 'EXPNAME' in list(headerObj.header.keys()):
                         self.expName = headerObj.header['EXPNAME']
                     if 'DATATYPE' in list(headerObj.header.keys()):
                         self.dataType = headerObj.header['DATATYPE']
                     self.datetimestr = headerObj.header['DATETIME']
                     channelList = headerObj.header['CHANNELLIST']
                     channelList = channelList.split('[')
                     channelList = channelList[1].split(']')
                     channelList = channelList[0].split(',')
                     channelList = [int(ch) for ch in channelList]
                     self.channelList = channelList
                     self.nChannels = headerObj.header['NCHANNELS']
                     self.nHeights = headerObj.header['NHEIGHTS']
                     self.ippSeconds = headerObj.header['IPPSECONDS']
                     self.nCohInt = headerObj.header['NCOHINT']
                     self.nIncohInt = headerObj.header['NINCOHINT']
                     self.dataBlocksPerFile = headerObj.header['NBLOCK']
                     self.timeZone = headerObj.header['TIMEZONE']
             #         self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     if 'COMMENT' in list(headerObj.header.keys()):
                         self.comments = headerObj.header['COMMENT']
                     self.readHeightList()
                 def readHeightList(self):
                     self.blockIndex = self.blockIndex + 1
                     obj = self.fitsObj[self.blockIndex]
                     self.heightList = obj.data
                     self.blockIndex = self.blockIndex + 1
                 def readExtension(self):
                     obj = self.fitsObj[self.blockIndex]
                     self.heightList = obj.data
                     self.blockIndex = self.blockIndex + 1
                 def setNextFile(self):
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         return 0
                     self.readHeader()
                     self.__setValuesFromHeader()
                     self.nReadBlocks = 0
             #         self.blockIndex = 1
                     return 1
                 def searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.fits',
                                         walk=True):
                     pathList = []
                     if not walk:
                         pathList.append(path)
                     else:
                         dirList = []
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isRadarFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None
                         thisDate = startDate
                         while(thisDate <= endDate):
                             year = thisDate.timetuple().tm_year
                             doy = thisDate.timetuple().tm_yday
                             matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
                             if len(matchlist) == 0:
                                 thisDate += datetime.timedelta(1)
                                 continue
                             for match in matchlist:
                                 pathList.append(os.path.join(path,match,expLabel))
                             thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print("Any folder was found for the date range: %s-%s" %(startDate, endDate))
                         return None, None
                     print("%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate))
                     filenameList = []
                     datetimeList = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for thisFile in fileList:
                             filename = os.path.join(thisPath,thisFile)
                             thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print("Any file was found for the time range %s - %s" %(startTime, endTime))
                         return None, None
                     print("%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime))
                     print()
                     for i in range(len(filenameList)):
                         print("%s -> [%s]" %(filenameList[i], datetimeList[i].ctime()))
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def setup(self, path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=0,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True):
                     if path == None:
                         raise ValueError("The path is not valid")
                     if ext == None:
                         ext = self.ext
                     if not(online):
                         print("Searching files in offline mode ...")
                         pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk)
                         if not(pathList):
                             print("No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime()))
                             sys.exit(-1)
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                     self.online = online
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print("No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
                         elif startDate != None:
                             print("No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime()))
                         else:
                             print("No files")
                         sys.exit(-1)
                 def readBlock(self):
                     dataObj = self.fitsObj[self.blockIndex]
                     self.data = dataObj.data
                     self.data_header_dict = dataObj.header
                     self.utc = self.data_header_dict['UTCTIME']
                     self.flagIsNewFile = 0
                     self.blockIndex += 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def __jumpToLastBlock(self):
                     raise NotImplementedError
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.__rdBasicHeader()
                             return 1
                         print("\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1))
                         sleep( self.delay )
                     return 0
                 def __setNewBlock(self):
                     if self.online:
                         self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         return 1
                     self.lastUTTime = self.utc
                     if self.online:
                         if self.__waitNewBlock():
                             return 1
                     if self.nReadBlocks < self.dataBlocksPerFile:
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.utc - self.lastUTTime
                     self.flagDiscontinuousBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagDiscontinuousBlock = 1
                     return 1
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def printInfo(self):
                     pass
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         return (0, 'No more files')
                     self.flagDiscontinuousBlock = 0
                     self.flagIsNewBlock = 0
                     if not(self.readNextBlock()):
                         return (1, 'Error reading data')
                     if self.data is None:
                         self.dataOut.flagNoData = True
                         return (0, 'No more data')
                     self.dataOut.data = self.data
                     self.dataOut.data_header = self.data_header_dict
                     self.dataOut.utctime = self.utc
             #         self.dataOut.header = self.header_dict
             #         self.dataOut.expName = self.expName
             #         self.dataOut.nChannels = self.nChannels
             #         self.dataOut.timeZone = self.timeZone
             #         self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
             #         self.dataOut.comments = self.comments
             # #         self.dataOut.timeInterval = self.timeInterval
             #         self.dataOut.channelList = self.channelList
             #         self.dataOut.heightList = self.heightList
                     self.dataOut.flagNoData = False
                     # return self.dataOut.data
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             @MPDecorator
             class SpectraHeisWriter(Operation):
             #    set = None
                 setFile = None
                 idblock = None
                 doypath = None
                 subfolder = None
                 def __init__(self):#, **kwargs):
                     Operation.__init__(self)#, **kwargs)
                     self.wrObj = PyFits()
             #        self.dataOut = dataOut
                     self.nTotalBlocks=0
             #        self.set = None
                     self.setFile = None
                     self.idblock = 0
                     self.wrpath = None
                     self.doypath = None
                     self.subfolder = None
                     self.isConfig = False
                 def isNumber(str):
                     """
                     Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                     Excepciones:
                     Si un determinado string no puede ser convertido a numero
                     Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                     Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                     """
                     try:
                         float( str )
                         return True
                     except:
                         return False
                 def setup(self, dataOut, wrpath):
                     if not(os.path.exists(wrpath)):
                         os.mkdir(wrpath)
                     self.wrpath = wrpath
             #        self.setFile = 0
                     self.dataOut = dataOut
                 def putData(self):
                     name= time.localtime( self.dataOut.utctime)
                     ext=".fits"
                     if self.doypath == None:
                        self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
                        self.doypath = os.path.join( self.wrpath, self.subfolder )
                        os.mkdir(self.doypath)
                     if self.setFile == None:
             #           self.set = self.dataOut.set
                        self.setFile = 0
             #        if self.set != self.dataOut.set:
             ##            self.set = self.dataOut.set
             #            self.setFile = 0
                     #make the filename
                     thisFile = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
                     filename = os.path.join(self.wrpath,self.subfolder, thisFile)
                     idblock = numpy.array([self.idblock],dtype="int64")
                     header=self.wrObj.cFImage(idblock=idblock,
                                             year=time.gmtime(self.dataOut.utctime).tm_year,
                                             month=time.gmtime(self.dataOut.utctime).tm_mon,
                                             day=time.gmtime(self.dataOut.utctime).tm_mday,
                                             hour=time.gmtime(self.dataOut.utctime).tm_hour,
                                             minute=time.gmtime(self.dataOut.utctime).tm_min,
                                             second=time.gmtime(self.dataOut.utctime).tm_sec)
                     c=3E8
                     deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
                     colList = []
                     colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
                     colList.append(colFreq)
                     nchannel=self.dataOut.nChannels
                     for i in range(nchannel):
                         col = self.wrObj.writeData(name="PCh"+str(i+1),
                                                      format=str(self.dataOut.nFFTPoints)+'E',
                                                       data=10*numpy.log10(self.dataOut.data_spc[i,:]))
                         colList.append(col)
                     data=self.wrObj.Ctable(colList=colList)
                     self.wrObj.CFile(header,data)
                     self.wrObj.wFile(filename)
                     #update the setFile
                     self.setFile += 1
                     self.idblock += 1
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
                     return dataOut

schainpy/model/io/jroIO_madrigal.py +1 -1

             '''
             Created on Aug 1, 2017
             @author: Juan C. Espinoza
             '''
             import os
             import sys
             import time
             import json
             import glob
             import datetime
             import numpy
             import h5py
             import schainpy.admin
             from schainpy.model.io.jroIO_base import LOCALTIME, Reader
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jrodata import Parameters
             from schainpy.utils import log
             try:
                 import madrigal.cedar
             except:
                 pass
             try:
                 basestring
             except:
                 basestring = str
             DEF_CATALOG = {
                 'principleInvestigator': 'Marco Milla',
                 'expPurpose': '',
                 'cycleTime': '',
                 'correlativeExp': '',
                 'sciRemarks': '',
                 'instRemarks': ''
                 }
             DEF_HEADER = {
                 'kindatDesc': '',
                 'analyst': 'Jicamarca User',
                 'comments': '',
                 'history': ''
                 }
             MNEMONICS = {
 : 'jro',
 : 'jbr',
 : 'jul',
 : 'jas',
 : 'pbr',
 : 'hbr',
 : 'obr',
 : 'clr'
             }
             UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
             def load_json(obj):
                 '''
                 Parse json as string instead of unicode
                 '''
                 if isinstance(obj, str):
                     iterable = json.loads(obj)
                 else:
                     iterable = obj
                 if isinstance(iterable, dict):
                     return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, basestring) else v
                         for k, v in list(iterable.items())}
                 elif isinstance(iterable, (list, tuple)):
                     return [str(v) if isinstance(v, basestring) else v for v in iterable]
                 return iterable
-            @MPDecorator
             class MADReader(Reader, ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                     self.counter_records = 0
                     self.nrecords = None
                     self.flagNoMoreFiles = 0
                     self.filename = None
                     self.intervals = set()
                     self.datatime = datetime.datetime(1900,1,1)
                     self.format = None
                     self.filefmt = "***%Y%m%d*******"
                 def setup(self, **kwargs):
                     self.set_kwargs(**kwargs)
                     self.oneDDict = load_json(self.oneDDict)
                     self.twoDDict = load_json(self.twoDDict)
                     self.ind2DList = load_json(self.ind2DList)
                     self.independentParam = self.ind2DList[0]
                     if self.path is None:
                         raise ValueError('The path is not valid')
                     self.open_file = open
                     self.open_mode = 'rb'
                     if self.format is None:
                         raise ValueError('The format is not valid choose simple or hdf5')
                     elif self.format.lower() in ('simple', 'txt'):
                         self.ext = '.txt'
                     elif self.format.lower() in ('cedar',):
                         self.ext = '.001'
                     else:
                         self.ext = '.hdf5'
                         self.open_file = h5py.File
                         self.open_mode = 'r'
                     if self.online:
                         log.log("Searching files in online mode...", self.name)
                         for nTries in range(self.nTries):
                             fullpath = self.searchFilesOnLine(self.path, self.startDate,
                                 self.endDate, self.expLabel, self.ext, self.walk,
                                 self.filefmt, self.folderfmt)
                             try:
                                 fullpath = next(fullpath)
                             except:
                                 fullpath = None
                             if fullpath:
                                 break
                             log.warning(
                                 'Waiting {} sec for a valid file in {}: try {} ...'.format(
                                     self.delay, self.path, nTries + 1),
                                 self.name)
                             time.sleep(self.delay)
                         if not(fullpath):
                             raise schainpy.admin.SchainError(
                                 'There isn\'t any valid file in {}'.format(self.path))
                     else:
                         log.log("Searching files in {}".format(self.path), self.name)
                         self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
                             self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
                     self.setNextFile()
                 def readFirstHeader(self):
                     '''Read header and data'''
                     self.parseHeader()
                     self.parseData()
                     self.blockIndex = 0
                     return
                 def parseHeader(self):
                     '''
                     '''
                     self.output = {}
                     self.version = '2'
                     s_parameters = None
                     if self.ext == '.txt':
                         self.parameters = [s.strip().lower() for s in self.fp.readline().decode().strip().split(' ') if s]
                     elif self.ext == '.hdf5':
                         self.metadata = self.fp['Metadata']
                         if '_record_layout' in self.metadata:
                             s_parameters = [s[0].lower().decode() for s in self.metadata['Independent Spatial Parameters']]
                             self.version = '3'
                         self.parameters = [s[0].lower().decode() for s in self.metadata['Data Parameters']]
                     log.success('Parameters found: {}'.format(self.parameters),
                                 'MADReader')
                     if s_parameters:
                         log.success('Spatial parameters found: {}'.format(s_parameters),
                                     'MADReader')
                     for param in list(self.oneDDict.keys()):
                         if param.lower() not in self.parameters:
                             log.warning(
                                 'Parameter {} not found will be ignored'.format(
                                     param),
                                 'MADReader')
                             self.oneDDict.pop(param, None)
                     for param, value in list(self.twoDDict.items()):
                         if param.lower() not in self.parameters:
                             log.warning(
                                 'Parameter {} not found, it will be ignored'.format(
                                     param),
                                 'MADReader')
                             self.twoDDict.pop(param, None)
                             continue
                         if isinstance(value, list):
                             if value[0] not in self.output:
                                 self.output[value[0]] = []
                             self.output[value[0]].append([])
                 def parseData(self):
                     '''
                     '''
                     if self.ext == '.txt':
                         self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
                         self.nrecords = self.data.shape[0]
                         self.ranges = numpy.unique(self.data[:,self.parameters.index(self.independentParam.lower())])
                         self.counter_records = 0
                     elif self.ext == '.hdf5':
                         self.data = self.fp['Data']
                         self.ranges = numpy.unique(self.data['Table Layout'][self.independentParam.lower()])
                         self.times = numpy.unique(self.data['Table Layout']['ut1_unix'])
                         self.counter_records = int(self.data['Table Layout']['recno'][0])
                         self.nrecords = int(self.data['Table Layout']['recno'][-1])
                 def readNextBlock(self):
                     while True:
                         self.flagDiscontinuousBlock = 0
                         if self.counter_records == self.nrecords:
                             self.setNextFile()
                         self.readBlock()
                         if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
                            (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
                             log.warning(
                                 'Reading Record No. {}/{} -> {} [Skipping]'.format(
                                     self.counter_records,
                                     self.nrecords,
                                     self.datatime.ctime()),
                                 'MADReader')
                             continue
                         break
                     log.log(
                         'Reading Record No. {}/{} -> {}'.format(
                             self.counter_records,
                             self.nrecords,
                             self.datatime.ctime()),
                         'MADReader')
                     return 1
                 def readBlock(self):
                     '''
                     '''
                     dum = []
                     if self.ext == '.txt':
                         dt = self.data[self.counter_records][:6].astype(int)
                         if datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]).date() > self.datatime.date():
                             self.flagDiscontinuousBlock = 1
                         self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
                         while True:
                             dt = self.data[self.counter_records][:6].astype(int)
                             datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
                             if datatime == self.datatime:
                                 dum.append(self.data[self.counter_records])
                                 self.counter_records += 1
                                 if self.counter_records == self.nrecords:
                                     break
                                 continue
                             self.intervals.add((datatime-self.datatime).seconds)
                             break
                     elif self.ext == '.hdf5':
                         datatime = datetime.datetime.utcfromtimestamp(
                             self.times[self.counter_records])
                         dum = self.data['Table Layout'][self.data['Table Layout']['recno']==self.counter_records]
                         self.intervals.add((datatime-self.datatime).seconds)
                         if datatime.date()>self.datatime.date():
                             self.flagDiscontinuousBlock = 1
                         self.datatime = datatime
                         self.counter_records += 1
                     self.buffer = numpy.array(dum)
                     return
                 def set_output(self):
                     '''
                     Storing data from buffer to dataOut object
                     '''
                     parameters = [None for __ in self.parameters]
                     for param, attr in list(self.oneDDict.items()):
                         x = self.parameters.index(param.lower())
                         setattr(self.dataOut, attr, self.buffer[0][x])
                     for param, value in list(self.twoDDict.items()):
                         dummy = numpy.zeros(self.ranges.shape) + numpy.nan
                         if self.ext == '.txt':
                             x = self.parameters.index(param.lower())
                             y = self.parameters.index(self.independentParam.lower())
                             ranges = self.buffer[:,y]
                             #if self.ranges.size == ranges.size:
                             #    continue
                             index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
                             dummy[index] = self.buffer[:,x]
                         else:
                             ranges = self.buffer[self.independentParam.lower()]
                             index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
                             dummy[index] = self.buffer[param.lower()]
                         if isinstance(value, str):
                             if value not in self.independentParam:
                                 setattr(self.dataOut, value, dummy.reshape(1,-1))
                         elif isinstance(value, list):
                             self.output[value[0]][value[1]] = dummy
                             parameters[value[1]] = param
                     for key, value in list(self.output.items()):
                         setattr(self.dataOut, key, numpy.array(value))
                     self.dataOut.parameters = [s for s in parameters if s]
                     self.dataOut.heightList = self.ranges
                     self.dataOut.utctime = (self.datatime - datetime.datetime(1970, 1, 1)).total_seconds()
                     self.dataOut.utctimeInit = self.dataOut.utctime
                     self.dataOut.paramInterval = min(self.intervals)
                     self.dataOut.useLocalTime = False
                     self.dataOut.flagNoData = False
                     self.dataOut.nrecords = self.nrecords
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                 def getData(self):
                     '''
                     Storing data from databuffer to dataOut object
                     '''
                     if not  self.readNextBlock():
                         self.dataOut.flagNoData = True
                         return 0
                     self.set_output()
                     return 1
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
                     return
             @MPDecorator
             class MADWriter(Operation):
                 '''Writing module for Madrigal files
             type: external
             Inputs:
                         path        path where files will be created
                         oneDDict    json of one-dimensional parameters in record where keys
                                     are Madrigal codes (integers or mnemonics) and values the corresponding
                                     dataOut attribute e.g: {
                                         'gdlatr': 'lat',
                                         'gdlonr': 'lon',
                                         'gdlat2':'lat',
                                         'glon2':'lon'}
                         ind2DList   list of independent spatial two-dimensional parameters e.g:
                                     ['heigthList']
                         twoDDict    json of two-dimensional parameters in record where keys
                                     are Madrigal codes (integers or mnemonics) and values the corresponding
                                     dataOut attribute if multidimensional array specify as tupple
                                     ('attr', pos) e.g: {
                                         'gdalt': 'heightList',
                                         'vn1p2': ('data_output', 0),
                                         'vn2p2': ('data_output', 1),
                                         'vn3': ('data_output', 2),
                                         'snl': ('data_SNR', 'db')
                                         }
                         metadata    json of madrigal metadata (kinst, kindat, catalog and header)
                         format      hdf5, cedar
                         blocks      number of blocks per file'''
                 __attrs__ = ['path', 'oneDDict', 'ind2DList', 'twoDDict','metadata', 'format', 'blocks']
                 missing = -32767
                 def __init__(self):
                     Operation.__init__(self)
                     self.dataOut = Parameters()
                     self.counter = 0
                     self.path = None
                     self.fp = None
                 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
                         metadata='{}', format='cedar', **kwargs):
                     if not self.isConfig:
                         self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
                         self.isConfig = True
                     self.dataOut = dataOut
                     self.putData()
                     return 1
                 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
                     '''
                     Configure Operation
                     '''
                     self.path = path
                     self.blocks = kwargs.get('blocks', None)
                     self.counter = 0
                     self.oneDDict = load_json(oneDDict)
                     self.twoDDict = load_json(twoDDict)
                     self.ind2DList = load_json(ind2DList)
                     meta = load_json(metadata)
                     self.kinst = meta.get('kinst')
                     self.kindat = meta.get('kindat')
                     self.catalog = meta.get('catalog', DEF_CATALOG)
                     self.header = meta.get('header', DEF_HEADER)
                     if format == 'cedar':
                         self.ext = '.dat'
                         self.extra_args = {}
                     elif format == 'hdf5':
                         self.ext = '.hdf5'
                         self.extra_args = {'ind2DList': self.ind2DList}
                     self.keys = [k.lower() for k in self.twoDDict]
                     if 'range' in self.keys:
                         self.keys.remove('range')
                     if 'gdalt' in self.keys:
                         self.keys.remove('gdalt')
                 def setFile(self):
                     '''
                     Create new cedar file object
                     '''
                     self.mnemonic = MNEMONICS[self.kinst]   #TODO get mnemonic from madrigal
                     date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
                     filename = '{}{}{}'.format(self.mnemonic,
                                                date.strftime('%Y%m%d_%H%M%S'),
                                                self.ext)
                     self.fullname = os.path.join(self.path, filename)
                     if os.path.isfile(self.fullname) :
                         log.warning(
                             'Destination file {} already exists, previous file deleted.'.format(
                                 self.fullname),
                             'MADWriter')
                         os.remove(self.fullname)
                     try:
                         log.success(
                             'Creating file: {}'.format(self.fullname),
                             'MADWriter')
                         if not os.path.exists(self.path):
                             os.makedirs(self.path)
                         self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
                     except ValueError as e:
                         log.error(
                             'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
                             'MADWriter')
                         return
                     return 1
                 def writeBlock(self):
                     '''
                     Add data records to cedar file taking data from oneDDict and twoDDict
                     attributes.
                     Allowed parameters in: parcodes.tab
                     '''
                     startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
                     endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
                     heights = self.dataOut.heightList
                     if self.ext == '.dat':
                         for key, value in list(self.twoDDict.items()):
                             if isinstance(value, str):
                                 data = getattr(self.dataOut, value)
                                 invalid = numpy.isnan(data)
                                 data[invalid] = self.missing
                             elif isinstance(value, (tuple, list)):
                                 attr, key = value
                                 data = getattr(self.dataOut, attr)
                                 invalid = numpy.isnan(data)
                                 data[invalid] = self.missing
                     out = {}
                     for key, value in list(self.twoDDict.items()):
                         key = key.lower()
                         if isinstance(value, str):
                             if 'db' in value.lower():
                                 tmp = getattr(self.dataOut, value.replace('_db', ''))
                                 SNRavg = numpy.average(tmp, axis=0)
                                 tmp = 10*numpy.log10(SNRavg)
                             else:
                                 tmp = getattr(self.dataOut, value)
                             out[key] = tmp.flatten()[:len(heights)]
                         elif isinstance(value, (tuple, list)):
                             attr, x = value
                             data = getattr(self.dataOut, attr)
                             out[key] = data[int(x)][:len(heights)]
                     a = numpy.array([out[k] for k in self.keys])
                     nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
                     index = numpy.where(nrows == False)[0]
                     rec = madrigal.cedar.MadrigalDataRecord(
                         self.kinst,
                         self.kindat,
                         startTime.year,
                         startTime.month,
                         startTime.day,
                         startTime.hour,
                         startTime.minute,
                         startTime.second,
                         startTime.microsecond/10000,
                         endTime.year,
                         endTime.month,
                         endTime.day,
                         endTime.hour,
                         endTime.minute,
                         endTime.second,
                         endTime.microsecond/10000,
                         list(self.oneDDict.keys()),
                         list(self.twoDDict.keys()),
                         len(index),
                         **self.extra_args
                     )
                     # Setting 1d values
                     for key in self.oneDDict:
                         rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
                     # Setting 2d values
                     nrec = 0
                     for n in index:
                         for key in out:
                             rec.set2D(key, nrec, out[key][n])
                         nrec += 1
                     self.fp.append(rec)
                     if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
                         self.fp.dump()
                     if self.counter % 20 == 0 and self.counter > 0:
                         log.log(
                             'Writing {} records'.format(
                                 self.counter),
                             'MADWriter')
                 def setHeader(self):
                     '''
                     Create an add catalog and header to cedar file
                     '''
                     log.success('Closing file {}'.format(self.fullname), 'MADWriter')
                     if self.ext == '.dat':
                         self.fp.write()
                     else:
                         self.fp.dump()
                         self.fp.close()
                     header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
                     header.createCatalog(**self.catalog)
                     header.createHeader(**self.header)
                     header.write()
                 def putData(self):
                     if self.dataOut.flagNoData:
                         return 0
                     if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
                         if self.counter > 0:
                             self.setHeader()
                         self.counter = 0
                     if self.counter == 0:
                         self.setFile()
                     self.writeBlock()
                     self.counter += 1
                 def close(self):
                     if self.counter > 0:
                         self.setHeader()

schainpy/model/io/jroIO_param.py +2 -2

             import numpy
             import time
             import os
             import h5py
             import re
             import datetime
             import schainpy.admin
             from schainpy.model.data.jrodata import *
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.io.jroIO_base import *
             from schainpy.utils import log
-            @MPDecorator
             class ParamReader(JRODataReader,ProcessingUnit):
                 '''
                 Reads HDF5 format files
                 path
                 startDate
                 endDate
                 startTime
                 endTime
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 timezone = None
                 startTime = None
                 endTime = None
                 fileIndex = None
                 utcList = None      #To select data in the utctime list
                 blockList = None    #List to blocks to be read from the file
                 blocksPerFile = None    #Number of blocks to be read
                 blockIndex = None
                 path = None
                 #List of Files
                 filenameList = None
                 datetimeList = None
                 #Hdf5 File
                 listMetaname = None
                 listMeta = None
                 listDataname = None
                 listData = None
                 listShapes = None
                 fp = None
                 #dataOut reconstruction
                 dataOut = None
                 def __init__(self):#, **kwargs):
                     ProcessingUnit.__init__(self) #, **kwargs)
                     self.dataOut = Parameters()
                     return
                 def setup(self, **kwargs):
                     path = kwargs['path']
                     startDate = kwargs['startDate']
                     endDate = kwargs['endDate']
                     startTime = kwargs['startTime']
                     endTime = kwargs['endTime']
                     walk = kwargs['walk']
                     if 'ext' in kwargs:
                         ext = kwargs['ext']
                     else:
                         ext = '.hdf5'
                     if 'timezone' in kwargs:
                         self.timezone = kwargs['timezone']
                     else:
                         self.timezone = 'lt'
                     print("[Reading] Searching files in offline mode ...")
                     pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            ext=ext, walk=walk)
                     if not(filenameList):
                         print("There is no files into the folder: %s"%(path))
                         sys.exit(-1)
                     self.fileIndex = -1
                     self.startTime = startTime
                     self.endTime = endTime
                     self.__readMetadata()
                     self.__setNextFileOffline()
                     return
                 def searchFilesOffLine(self,
                                         path,
                                         startDate=None,
                                         endDate=None,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         ext='.hdf5',
                                         walk=True):
                     expLabel = ''
                     self.filenameList = []
                     self.datetimeList = []
                     pathList = []
                     JRODataObj = JRODataReader()
                     dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
                     if dateList == []:
                         print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime()))
                         return None, None
                     if len(dateList) > 1:
                         print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
                     else:
                         print("[Reading] data was found for the date %s" %(dateList[0]))
                     filenameList = []
                     datetimeList = []
                     #----------------------------------------------------------------------------------
                     for thisPath in pathList:
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             if not isFileInDateRange(filename, startDate, endDate):
                                 continue
                             thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
                         return None, None
                     print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
                     print()
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
                     """
                     Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                     Inputs:
                         filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                         startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                         endDate            :    fecha final del rango seleccionado en formato datetime.date
                         startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                         endTime            :    tiempo final del rango seleccionado en formato datetime.time
                     Return:
                         Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                         fecha especificado, de lo contrario retorna False.
                     Excepciones:
                         Si el archivo no existe o no puede ser abierto
                         Si la cabecera no puede ser leida.
                     """
                     try:
                         fp = h5py.File(filename,'r')
                         grp1 = fp['Data']
                     except IOError:
                         traceback.print_exc()
                         raise IOError("The file %s can't be opened" %(filename))
                     #In case has utctime attribute
                     grp2 = grp1['utctime']
              #         thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
                     thisUtcTime = grp2.value[0]
                     fp.close()
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
                     #General case
                     #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                     #-----------o----------------------------o-----------
                     #       startTime                     endTime
                     if endTime >= startTime:
                         thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
                         if numpy.any(thisUtcLog):   #If there is one block between the hours mentioned
                             return thisDatetime
                         return None
                     #If endTime < startTime then endTime belongs to the next day
                     #<<<<<<<<<<<o                            o>>>>>>>>>>>
                     #-----------o----------------------------o-----------
                     #        endTime                    startTime
                     if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
                         return None
                     if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
                         return None
                     if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
                         return None
                     return thisDatetime
                 def __setNextFileOffline(self):
                     self.fileIndex += 1
                     idFile = self.fileIndex
                     if not(idFile < len(self.filenameList)):
                         raise schainpy.admin.SchainError("No more Files")
                         return 0
                     filename = self.filenameList[idFile]
                     filePointer = h5py.File(filename,'r')
                     self.filename = filename
                     self.fp = filePointer
                     print("Setting the file: %s"%self.filename)
                     self.__setBlockList()
                     self.__readData()
                     self.blockIndex = 0
                     return 1
                 def __setBlockList(self):
                     '''
                     Selects the data within the times defined
                     self.fp
                     self.startTime
                     self.endTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     fp = self.fp
                     startTime = self.startTime
                     endTime = self.endTime
                     grp = fp['Data']
                     thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
                     #ERROOOOR
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     return
                 def __readMetadata(self):
                     '''
                     Reads Metadata
                     self.pathMeta
                     self.listShapes
                     self.listMetaname
                     self.listMeta
                     '''
                     filename = self.filenameList[0]
                     fp = h5py.File(filename,'r')
                     gp = fp['Metadata']
                     listMetaname = []
                     listMetadata = []
                     for item in list(gp.items()):
                         name = item[0]
                         if name=='array dimensions':
                             table = gp[name][:]
                             listShapes = {}
                             for shapes in table:
                                 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
                         else:
                             data = gp[name].value
                             listMetaname.append(name)
                             listMetadata.append(data)
                     self.listShapes = listShapes
                     self.listMetaname = listMetaname
                     self.listMeta = listMetadata
                     fp.close()
                     return
                 def __readData(self):
                     grp = self.fp['Data']
                     listdataname = []
                     listdata = []
                     for item in list(grp.items()):
                         name = item[0]
                         listdataname.append(name)
                         array = self.__setDataArray(grp[name],self.listShapes[name])
                         listdata.append(array)
                     self.listDataname = listdataname
                     self.listData = listdata
                     return
                 def __setDataArray(self, dataset, shapes):
                     nDims = shapes[0]
                     nDim2 = shapes[1]      #Dimension 0
                     nDim1 = shapes[2]      #Dimension 1, number of Points or Parameters
                     nDim0 = shapes[3]      #Dimension 2, number of samples or ranges
                     mode = shapes[4]        #Mode of storing
                     blockList = self.blockList
                     blocksPerFile = self.blocksPerFile
                     #Depending on what mode the data was stored
                     if mode == 0:       #Divided in channels
                         arrayData = dataset.value.astype(numpy.float)[0][blockList]
                     if mode == 1:     #Divided in parameter
                         strds = 'table'
                         nDatas = nDim1
                         newShapes = (blocksPerFile,nDim2,nDim0)
                     elif mode==2:       #Concatenated in a table
                         strds = 'table0'
                         arrayData = dataset[strds].value
                         #Selecting part of the dataset
                         utctime = arrayData[:,0]
                         u, indices = numpy.unique(utctime, return_index=True)
                         if blockList.size != indices.size:
                             indMin = indices[blockList[0]]
                             if blockList[1] + 1 >= indices.size:
                                 arrayData = arrayData[indMin:,:]
                             else:
                                 indMax = indices[blockList[1] + 1]
                                 arrayData = arrayData[indMin:indMax,:]
                         return arrayData
                     #    One dimension
                     if nDims == 0:
                         arrayData = dataset.value.astype(numpy.float)[0][blockList]
                     #    Two dimensions
                     elif nDims == 2:
                         arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
                         newShapes = (blocksPerFile,nDim0)
                         nDatas = nDim1
                         for i in range(nDatas):
                             data = dataset[strds + str(i)].value
                             arrayData[:,i,:] = data[blockList,:]
                     #    Three dimensions
                     else:
                         arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
                         for i in range(nDatas):
                             data = dataset[strds + str(i)].value
                             for b in range(blockList.size):
                                 arrayData[b,:,i,:] = data[:,:,blockList[b]]
                     return arrayData
                 def __setDataOut(self):
                     listMeta = self.listMeta
                     listMetaname = self.listMetaname
                     listDataname = self.listDataname
                     listData = self.listData
                     listShapes = self.listShapes
                     blockIndex = self.blockIndex
              #         blockList = self.blockList
                     for i in range(len(listMeta)):
                         setattr(self.dataOut,listMetaname[i],listMeta[i])
                     for j in range(len(listData)):
                         nShapes = listShapes[listDataname[j]][0]
                         mode = listShapes[listDataname[j]][4]
                         if nShapes == 1:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
                         elif nShapes > 1:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
                         elif mode==0:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
                         #Mode Meteors
                         elif mode ==2:
                             selectedData = self.__selectDataMode2(listData[j], blockIndex)
                             setattr(self.dataOut, listDataname[j], selectedData)
                     return
                 def __selectDataMode2(self, data, blockIndex):
                     utctime = data[:,0]
                     aux, indices = numpy.unique(utctime, return_inverse=True)
                     selInd = numpy.where(indices == blockIndex)[0]
                     selData = data[selInd,:]
                     return selData
                 def getData(self):
                     if self.blockIndex==self.blocksPerFile:
                          if not( self.__setNextFileOffline() ):
                             self.dataOut.flagNoData = True
                             return 0
                     self.__setDataOut()
                     self.dataOut.flagNoData = False
                     self.blockIndex += 1
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
                     return
             @MPDecorator
             class ParamWriter(Operation):
                 '''
                 HDF5 Writer, stores parameters data in HDF5 format files
                 path:             path where the files will be stored
                 blocksPerFile:    number of blocks that will be saved in per HDF5 format file
                 mode:             selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
                 metadataList:     list of attributes that will be stored as metadata
                 dataList:         list of attributes that will be stores as data
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 metaoptchar = "M"
                 metaFile = None
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 grp = None
                 ds = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 dsList = None   #List of dictionaries with dataset properties
                 tableDim = None
                 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
                 currentDay = None
                 lastTime = None
                 setType = None
                 def __init__(self):
                     Operation.__init__(self)
                     return
                 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataList = dataList
                     self.dataOut = dataOut
                     self.mode = mode
                     if self.mode is not None:
                         self.mode = numpy.zeros(len(self.dataList)) + mode
                     else:
                         self.mode = numpy.ones(len(self.dataList))
                     self.setType = setType
                     arrayDim = numpy.zeros((len(self.dataList),5))
                     #Table dimensions
                     dtype0 = self.dtype
                     tableList = []
                     #Dictionary and list of tables
                     dsList = []
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         dataAux = getattr(self.dataOut, self.dataList[i])
                         dsDict['variable'] = self.dataList[i]
                         #---------------------    Conditionals    ------------------------
                         #There is no data
                         if dataAux is None:
                             return 0
                         if isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
                             dsDict['mode'] = 0
                             dsDict['nDim'] = 0
                             arrayDim[i,0] = 0
                             dsList.append(dsDict)
                         #Mode 2: meteors
                         elif self.mode[i] == 2:
                             dsDict['dsName'] = 'table0'
                             dsDict['mode'] = 2      # Mode meteors
                             dsDict['shape'] = dataAux.shape[-1]
                             dsDict['nDim'] = 0
                             dsDict['dsNumber'] = 1
                             arrayDim[i,3] = dataAux.shape[-1]
                             arrayDim[i,4] = self.mode[i]         #Mode the data was stored
                             dsList.append(dsDict)
                         #Mode 1
                         else:
                             arrayDim0 = dataAux.shape       #Data dimensions
                             arrayDim[i,0] = len(arrayDim0)  #Number of array dimensions
                             arrayDim[i,4] = self.mode[i]         #Mode the data was stored
                             strtable = 'table'
                             dsDict['mode'] = 1      # Mode parameters
                             # Three-dimension arrays
                             if len(arrayDim0) == 3:
                                 arrayDim[i,1:-1] = numpy.array(arrayDim0)
                                 nTables = int(arrayDim[i,2])
                                 dsDict['dsNumber'] = nTables
                                 dsDict['shape'] = arrayDim[i,2:4]
                                 dsDict['nDim'] = 3
                                 for j in range(nTables):
                                     dsDict = dsDict.copy()
                                     dsDict['dsName'] = strtable + str(j)
                                     dsList.append(dsDict)
                             # Two-dimension arrays
                             elif len(arrayDim0) == 2:
                                 arrayDim[i,2:-1] = numpy.array(arrayDim0)
                                 nTables = int(arrayDim[i,2])
                                 dsDict['dsNumber'] = nTables
                                 dsDict['shape'] = arrayDim[i,3]
                                 dsDict['nDim'] = 2
                                 for j in range(nTables):
                                     dsDict = dsDict.copy()
                                     dsDict['dsName'] = strtable + str(j)
                                     dsList.append(dsDict)
                             # One-dimension arrays
                             elif len(arrayDim0) == 1:
                                 arrayDim[i,3] = arrayDim0[0]
                                 dsDict['shape'] = arrayDim0[0]
                                 dsDict['dsNumber'] = 1
                                 dsDict['dsName'] = strtable + str(0)
                                 dsDict['nDim'] = 1
                                 dsList.append(dsDict)
                         table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
                         tableList.append(table)
                     self.dsList = dsList
                     self.tableDim = numpy.array(tableList, dtype = dtype0)
                     self.blockIndex = 0
                     timeTuple = time.localtime(dataOut.utctime)
                     self.currentDay = timeTuple.tm_yday
                 def putMetadata(self):
                     fp = self.createMetadataFile()
                     self.writeMetadata(fp)
                     fp.close()
                     return
                 def createMetadataFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = ''
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         filesList = sorted( filesList, key=str.lower )
                         if len( filesList ) > 0:
                             filesList = [k for k in filesList if k.startswith(self.metaoptchar)]
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     filename = os.path.join( path, subfolder, file )
                     self.metaFile = file
                     #Setting HDF5 File
                     fp = h5py.File(filename,'w')
                     return fp
                 def writeMetadata(self, fp):
                     grp = fp.create_group("Metadata")
                     grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
                     for i in range(len(self.metadataList)):
                         grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
                     return
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     if self.lastTime is None:
                         self.lastTime = currentTime
                     #Day
                     timeTuple = time.localtime(currentTime)
                     dataDay = timeTuple.tm_yday
                     #Time
                     timeDiff = currentTime - self.lastTime
                     #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                     if dataDay != self.currentDay:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > 3*60*60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     mode = self.mode
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if os.path.exists(fullpath):
                         filesList = os.listdir( fullpath )
                         filesList = [k for k in filesList if 'M' in k]
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     filename = os.path.join( path, subfolder, file )
                     #Setting HDF5 File
                     fp = h5py.File(filename,'w')
                     #write metadata
                     self.writeMetadata(fp)
                     #Write data
                     grp = fp.create_group("Data")
                     ds = []
                     data = []
                     dsList = self.dsList
                     i = 0
                     while i < len(dsList):
                         dsInfo = dsList[i]
                         #One-dimension data
                         if dsInfo['mode'] == 0:
                             ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
                             ds.append(ds0)
                             data.append([])
                             i += 1
                             continue
                         elif dsInfo['mode'] == 2:
                             grp0 = grp.create_group(dsInfo['variable'])
                             ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
                             ds.append(ds0)
                             data.append([])
                             i += 1
                             continue
                         elif dsInfo['mode'] == 1:
                             grp0 = grp.create_group(dsInfo['variable'])
                             for j in range(dsInfo['dsNumber']):
                                 dsInfo = dsList[i]
                                 tableName = dsInfo['dsName']
                                 if dsInfo['nDim'] == 3:
                                     shape = dsInfo['shape'].astype(int)
                                     ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
                                 else:
                                     shape = int(dsInfo['shape'])
                                     ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
                                 ds.append(ds0)
                                 data.append([])
                                 i += 1
                     fp.flush()
                     fp.close()
                     log.log('creating file: {}'.format(filename), 'Writing')
                     self.filename = filename
                     self.ds = ds
                     self.data = data
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     if self.blockIndex == self.blocksPerFile or self.timeFlag():
                         self.setNextFile()
                     self.readBlock()
                     self.setBlock()     #Prepare data to be written
                     self.writeBlock()   #Write data
                     return
                 def readBlock(self):
                     '''
                     data Array configured
                     self.data
                     '''
                     dsList = self.dsList
                     ds = self.ds
                             #Setting HDF5 File
                     fp = h5py.File(self.filename,'r+')
                     grp = fp["Data"]
                     ind = 0
                     while ind < len(dsList):
                         dsInfo = dsList[ind]
                         if dsInfo['mode'] == 0:
                             ds0 = grp[dsInfo['variable']]
                             ds[ind] = ds0
                             ind += 1
                         else:
                             grp0 = grp[dsInfo['variable']]
                             for j in range(dsInfo['dsNumber']):
                                 dsInfo = dsList[ind]
                                 ds0 = grp0[dsInfo['dsName']]
                                 ds[ind] = ds0
                                 ind += 1
                     self.fp = fp
                     self.grp = grp
                     self.ds = ds
                     return
                 def setBlock(self):
                     '''
                     data Array configured
                     self.data
                     '''
                     #Creating Arrays
                     dsList = self.dsList
                     data = self.data
                     ind = 0
                     while ind < len(dsList):
                         dsInfo = dsList[ind]
                         dataAux = getattr(self.dataOut, dsInfo['variable'])
                         mode = dsInfo['mode']
                         nDim = dsInfo['nDim']
                         if mode == 0 or mode == 2 or nDim == 1:
                             data[ind] = dataAux
                             ind += 1
              #             elif nDim == 1:
              #                 data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
              #                 ind += 1
                         elif nDim == 2:
                             for j in range(dsInfo['dsNumber']):
                                 data[ind] = dataAux[j,:]
                                 ind += 1
                         elif nDim == 3:
                             for j in range(dsInfo['dsNumber']):
                                 data[ind] = dataAux[:,j,:]
                                 ind += 1
                     self.data = data
                     return
                 def writeBlock(self):
                     '''
                     Saves the block in the HDF5 file
                     '''
                     dsList = self.dsList
                     for i in range(len(self.ds)):
                         dsInfo = dsList[i]
                         nDim = dsInfo['nDim']
                         mode = dsInfo['mode']
                         #    First time
                         if self.firsttime:
                             if type(self.data[i]) == numpy.ndarray:
                                 if nDim == 3:
                                     self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
                                     self.ds[i].resize(self.data[i].shape)
                                 if mode == 2:
                                     self.ds[i].resize(self.data[i].shape)
                             self.ds[i][:] = self.data[i]
                         else:
                         #    From second time
                             #    Meteors!
                             if mode == 2:
                                 dataShape = self.data[i].shape
                                 dsShape = self.ds[i].shape
                                 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
                                 self.ds[i][dsShape[0]:,:] = self.data[i]
                             #    No dimension
                             elif mode == 0:
                                 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
                                 self.ds[i][0,-1] = self.data[i]
                             #    One dimension
                             elif nDim == 1:
                                 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
                                 self.ds[i][-1,:] = self.data[i]
                             #    Two dimension
                             elif nDim == 2:
                                 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
                                 self.ds[i][self.blockIndex,:] = self.data[i]
                             #    Three dimensions
                             elif nDim == 3:
                                 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
                                 self.ds[i][:,:,-1] = self.data[i]
                     self.firsttime = False
                     self.blockIndex += 1
                     #Close to save changes
                     self.fp.flush()
                     self.fp.close()
                     return
                 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
                     self.dataOut = dataOut
                     if not(self.isConfig):
                         self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList, mode=mode,
                                    setType=setType)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return
-            @MPDecorator
             class ParameterReader(Reader, ProcessingUnit):
                 '''
                 Reads HDF5 format files
                 '''
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                     self.ext = ".hdf5"
                     self.optchar = "D"
                     self.timezone = "lt"
                     self.listMetaname = []
                     self.listMeta = []
                     self.listDataname = []
                     self.listData = []
                     self.listShapes = []
                     self.open_file = h5py.File
                     self.open_mode = 'r'
                     self.metadata = False
                     self.filefmt = "*%Y%j***"
                     self.folderfmt = "*%Y%j"
                 def setup(self, **kwargs):
                     self.set_kwargs(**kwargs)
                     if not self.ext.startswith('.'):
                         self.ext = '.{}'.format(self.ext)
                     if self.online:
                         log.log("Searching files in online mode...", self.name)
                         for nTries in range(self.nTries):
                             fullpath = self.searchFilesOnLine(self.path, self.startDate,
                                 self.endDate, self.expLabel, self.ext, self.walk,
                                 self.filefmt, self.folderfmt)
                             try:
                                 fullpath = next(fullpath)
                             except:
                                 fullpath = None
                             if fullpath:
                                 break
                             log.warning(
                                 'Waiting {} sec for a valid file in {}: try {} ...'.format(
                                     self.delay, self.path, nTries + 1),
                                 self.name)
                             time.sleep(self.delay)
                         if not(fullpath):
                             raise schainpy.admin.SchainError(
                                 'There isn\'t any valid file in {}'.format(self.path))
                         pathname, filename = os.path.split(fullpath)
                         self.year = int(filename[1:5])
                         self.doy = int(filename[5:8])
                         self.set = int(filename[8:11]) - 1
                     else:
                         log.log("Searching files in {}".format(self.path), self.name)
                         self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
                             self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
                     self.setNextFile()
                     return
                 def readFirstHeader(self):
                     '''Read metadata and data'''
                     self.__readMetadata()
                     self.__readData()
                     self.__setBlockList()
                     self.blockIndex = 0
                     return
                 def __setBlockList(self):
                     '''
                     Selects the data within the times defined
                     self.fp
                     self.startTime
                     self.endTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     startTime = self.startTime
                     endTime = self.endTime
                     index = self.listDataname.index('utctime')
                     thisUtcTime = self.listData[index]
                     self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     return
                 def __readMetadata(self):
                     '''
                     Reads Metadata
                     '''
                     listMetaname = []
                     listMetadata = []
                     if 'Metadata' in self.fp:
                         gp = self.fp['Metadata']
                         for item in list(gp.items()):
                             name = item[0]
                             if name=='variables':
                                 table = gp[name][:]
                                 listShapes = {}
                                 for shapes in table:
                                     listShapes[shapes[0].decode()] = numpy.array([shapes[1]])
                             else:
                                 data = gp[name].value
                                 listMetaname.append(name)
                                 listMetadata.append(data)
                     elif self.metadata:
                         metadata = json.loads(self.metadata)
                         listShapes = {}
                         for tup in metadata:
                             name, values, dim = tup
                             if dim == -1:
                                 listMetaname.append(name)
                                 listMetadata.append(self.fp[values].value)
                             else:
                                 listShapes[name] = numpy.array([dim])
                     else:
                         raise IOError('Missing Metadata group in file or metadata info')
                     self.listShapes = listShapes
                     self.listMetaname = listMetaname
                     self.listMeta = listMetadata
                     return
                 def __readData(self):
                     listdataname = []
                     listdata = []
                     if 'Data' in self.fp:
                         grp = self.fp['Data']
                         for item in list(grp.items()):
                             name = item[0]
                             listdataname.append(name)
                             dim = self.listShapes[name][0]
                             if dim == 0:
                                 array = grp[name].value
                             else:
                                 array = []
                                 for i in range(dim):
                                     array.append(grp[name]['table{:02d}'.format(i)].value)
                                 array = numpy.array(array)
                             listdata.append(array)
                     elif self.metadata:
                         metadata = json.loads(self.metadata)
                         for tup in metadata:
                             name, values, dim = tup
                             listdataname.append(name)
                             if dim == -1:
                                 continue
                             elif dim == 0:
                                 array = self.fp[values].value
                             else:
                                 array = []
                                 for var in values:
                                     array.append(self.fp[var].value)
                                 array = numpy.array(array)
                             listdata.append(array)
                     else:
                         raise IOError('Missing Data group in file or metadata info')
                     self.listDataname = listdataname
                     self.listData = listdata
                     return
                 def getData(self):
                     for i in range(len(self.listMeta)):
                         setattr(self.dataOut, self.listMetaname[i], self.listMeta[i])
                     for j in range(len(self.listData)):
                         dim = self.listShapes[self.listDataname[j]][0]
                         if dim == 0:
                             setattr(self.dataOut, self.listDataname[j], self.listData[j][self.blockIndex])
                         else:
                             setattr(self.dataOut, self.listDataname[j], self.listData[j][:,self.blockIndex])
                     self.dataOut.paramInterval = self.interval
                     self.dataOut.flagNoData = False
                     self.blockIndex += 1
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     if self.blockIndex == self.blocksPerFile:
                         self.setNextFile()
                     self.getData()
                     return
             @MPDecorator
             class ParameterWriter(Operation):
                 '''
                 HDF5 Writer, stores parameters data in HDF5 format files
                 path:             path where the files will be stored
                 blocksPerFile:    number of blocks that will be saved in per HDF5 format file
                 mode:             selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
                 metadataList:     list of attributes that will be stored as metadata
                 dataList:         list of attributes that will be stores as data
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 metaoptchar = "M"
                 metaFile = None
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 grp = None
                 ds = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 dsList = None   #List of dictionaries with dataset properties
                 tableDim = None
                 dtype = [('name', 'S20'),('nDim', 'i')]
                 currentDay = None
                 lastTime = None
                 def __init__(self):
                     Operation.__init__(self)
                     return
                 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataList = dataList
                     self.setType = setType
                     tableList = []
                     dsList = []
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         dataAux = getattr(self.dataOut, self.dataList[i])
                         dsDict['variable'] = self.dataList[i]
                         if dataAux is None:
                             continue
                         elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
                             dsDict['nDim'] = 0
                         else:
                             dsDict['nDim'] = len(dataAux.shape)
                             dsDict['shape'] = dataAux.shape
                             dsDict['dsNumber'] = dataAux.shape[0]
                         dsList.append(dsDict)
                         tableList.append((self.dataList[i], dsDict['nDim']))
                     self.dsList = dsList
                     self.tableDim = numpy.array(tableList, dtype=self.dtype)
                     self.currentDay = self.dataOut.datatime.date()
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     timeTuple = time.localtime(currentTime)
                     dataDay = timeTuple.tm_yday
                     if self.lastTime is None:
                         self.lastTime = currentTime
                         self.currentDay = dataDay
                         return False
                     timeDiff = currentTime - self.lastTime
                     #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                     if dataDay != self.currentDay:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > 3*60*60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, setType=None):
                     self.dataOut = dataOut
                     if not(self.isConfig):
                         self.setup(path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList,
                                    setType=setType)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join(path, subfolder)
                     if os.path.exists(fullpath):
                         filesList = os.listdir(fullpath)
                         filesList = [k for k in filesList if k.startswith(self.optchar)]
                         if len( filesList ) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     self.filename = os.path.join( path, subfolder, file )
                     #Setting HDF5 File
                     self.fp = h5py.File(self.filename, 'w')
                     #write metadata
                     self.writeMetadata(self.fp)
                     #Write data
                     self.writeData(self.fp)
                 def writeMetadata(self, fp):
                     grp = fp.create_group("Metadata")
                     grp.create_dataset('variables', data=self.tableDim, dtype=self.dtype)
                     for i in range(len(self.metadataList)):
                         if not hasattr(self.dataOut, self.metadataList[i]):
                             log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
                             continue
                         value = getattr(self.dataOut, self.metadataList[i])
                         grp.create_dataset(self.metadataList[i], data=value)
                     return
                 def writeData(self, fp):
                     grp = fp.create_group("Data")
                     dtsets = []
                     data = []
                     for dsInfo in self.dsList:
                         if dsInfo['nDim'] == 0:
                             ds = grp.create_dataset(
                                 dsInfo['variable'],
                                 (self.blocksPerFile, ),
                                 chunks=True,
                                 dtype=numpy.float64)
                             dtsets.append(ds)
                             data.append((dsInfo['variable'], -1))
                         else:
                             sgrp = grp.create_group(dsInfo['variable'])
                             for i in range(dsInfo['dsNumber']):
                                 ds = sgrp.create_dataset(
                                     'table{:02d}'.format(i),
                                     (self.blocksPerFile, ) + dsInfo['shape'][1:],
                                     chunks=True)
                                 dtsets.append(ds)
                                 data.append((dsInfo['variable'], i))
                     fp.flush()
                     log.log('Creating file: {}'.format(fp.filename), self.name)
                     self.ds = dtsets
                     self.data = data
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
                         self.closeFile()
                         self.setNextFile()
                     for i, ds in enumerate(self.ds):
                         attr, ch = self.data[i]
                         if ch == -1:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)
                         else:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
                     self.fp.flush()
                     self.blockIndex += 1
                     log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
                     return
                 def closeFile(self):
                     if self.blockIndex != self.blocksPerFile:
                         for ds in self.ds:
                             ds.resize(self.blockIndex, axis=0)
                     self.fp.flush()
                     self.fp.close()
                 def close(self):
                     self.closeFile()

schainpy/model/io/jroIO_spectra.py +1 -1

             '''
             Created on Jul 2, 2014
             @author: roj-idl71
             '''
             import numpy
             from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jrodata import Spectra
             from schainpy.utils import log
-            @MPDecorator
             class SpectraReader(JRODataReader, ProcessingUnit):
                 """
                 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
                 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
                                     paresCanalesIguales    * alturas * perfiles  (Self Spectra)
                                     paresCanalesDiferentes * alturas * perfiles  (Cross Spectra)
                                     canales * alturas                            (DC Channels)
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = SpectraReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         readerObj.getData()
                         print readerObj.data_spc
                         print readerObj.data_cspc
                         print readerObj.data_dc
                         if readerObj.flagNoMoreFiles:
                             break
                 """
                 def __init__(self):#, **kwargs):
                     """
                     Inicializador de la clase SpectraReader para la lectura de datos de espectros.
                     Inputs:
                         dataOut    :    Objeto de la clase Spectra. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Affected:
                         self.dataOut
                     Return      : None
                     """
                     ProcessingUnit.__init__(self)
                     self.pts2read_SelfSpectra = 0
                     self.pts2read_CrossSpectra = 0
                     self.pts2read_DCchannels = 0
                     self.ext = ".pdata"
                     self.optchar = "P"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.lastUTTime = 0
                     self.maxTimeStep = 30
                     self.dataOut = Spectra()
                     self.profileIndex = 1
                     self.nRdChannels = None
                     self.nRdPairs = None
                     self.rdPairList = []
                 def createObjByDefault(self):
                     dataObj = Spectra()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     return 1
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.nRdChannels
                         self.nRdPairs
                         self.pts2read_SelfSpectra
                         self.pts2read_CrossSpectra
                         self.pts2read_DCchannels
                         self.blocksize
                         self.dataOut.nChannels
                         self.dataOut.nPairs
                     Return:
                         None
                     """
                     self.nRdChannels = 0
                     self.nRdPairs = 0
                     self.rdPairList = []
                     for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
                         if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
                             self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
                         else:
                             self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
                             self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
                     pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
                     self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
                     self.blocksize = self.pts2read_SelfSpectra
                     if self.processingHeaderObj.flag_cspc:
                         self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
                         self.blocksize += self.pts2read_CrossSpectra
                     if self.processingHeaderObj.flag_dc:
                         self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
                         self.blocksize += self.pts2read_DCchannels
                 def readBlock(self):
                     """
                     Lee el bloque de datos desde la posicion actual del puntero del archivo
                     (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                     (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                     es seteado a 0
                     Return: None
                     Variables afectadas:
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Exceptions:
                         Si un bloque leido no es un bloque valido
                     """
                     fpointer = self.fp.tell()
                     spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
                     spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
                         cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_dc:
                         dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
                         dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
                     if not self.processingHeaderObj.shif_fft:
                         #desplaza a la derecha en el eje 2 determinadas posiciones
                         shift = int(self.processingHeaderObj.profilesPerBlock/2)
                         spc = numpy.roll( spc, shift , axis=2 )
                         if self.processingHeaderObj.flag_cspc:
                             #desplaza a la derecha en el eje 2 determinadas posiciones
                             cspc = numpy.roll( cspc, shift, axis=2 )
                     #Dimensions : nChannels, nProfiles, nSamples
                     spc = numpy.transpose( spc, (0,2,1) )
                     self.data_spc = spc
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.transpose( cspc, (0,2,1) )
                         self.data_cspc = cspc['real'] + cspc['imag']*1j
                     else:
                         self.data_cspc = None
                     if self.processingHeaderObj.flag_dc:
                         self.data_dc = dc['real'] + dc['imag']*1j
                     else:
                         self.data_dc = None
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.getBasicHeader()
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
                     self.dataOut.dtype = self.dtype
                     self.dataOut.pairsList = self.rdPairList
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
                     xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
                     self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
                     self.dataOut.flagShiftFFT = True    #Data is always shifted
                     self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
                     self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
                 def getData(self):
                     """
                     First method to execute before "RUN" is called.
                     Copia el buffer de lectura a la clase "Spectra",
                     con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
                     Return:
 :    Si no hay mas archivos disponibles
 :    Si hizo una buena copia del buffer
                     Affected:
                         self.dataOut
                         self.flagDiscontinuousBlock
                         self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         return 0
                     self.flagDiscontinuousBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not( self.readNextBlock() ):
                             self.dataOut.flagNoData = True
                             return 0
                     #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
                     if self.data_spc is None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.getBasicHeader()
                     self.getFirstHeader()
                     self.dataOut.data_spc = self.data_spc
                     self.dataOut.data_cspc = self.data_cspc
                     self.dataOut.data_dc = self.data_dc
                     self.dataOut.flagNoData = False
                     self.dataOut.realtime = self.online
                     return self.dataOut.data_spc
             @MPDecorator
             class SpectraWriter(JRODataWriter, Operation):
                 """
                 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 def __init__(self):
                     """
                     Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj
                     Return: None
                     """
                     Operation.__init__(self)
                     self.ext = ".pdata"
                     self.optchar = "P"
                     self.shape_spc_Buffer = None
                     self.shape_cspc_Buffer = None
                     self.shape_dc_Buffer = None
                     self.data_spc = None
                     self.data_cspc = None
                     self.data_dc = None
                     self.setFile = None
                     self.noMoreFiles = 0
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     return 1
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shape_spc_Buffer = (self.dataOut.nChannels,
                                              self.processingHeaderObj.nHeights,
                                              self.processingHeaderObj.profilesPerBlock)
                     self.shape_cspc_Buffer = (self.dataOut.nPairs,
                                               self.processingHeaderObj.nHeights,
                                               self.processingHeaderObj.profilesPerBlock)
                     self.shape_dc_Buffer = (self.dataOut.nChannels,
                                             self.processingHeaderObj.nHeights)
                 def writeBlock(self):
                     """processingHeaderObj
                     Escribe el buffer en el file designado
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.nWriteBlocks
                     Return: None
                     """
                     spc = numpy.transpose( self.data_spc, (0,2,1) )
                     if not self.processingHeaderObj.shif_fft:
                         spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                     data = spc.reshape((-1))
                     data = data.astype(self.dtype[0])
                     data.tofile(self.fp)
                     if self.data_cspc is not None:
                         cspc = numpy.transpose( self.data_cspc, (0,2,1) )
                         data = numpy.zeros( numpy.shape(cspc), self.dtype )
                         #print 'data.shape', self.shape_cspc_Buffer
                         if not self.processingHeaderObj.shif_fft:
                             cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                         data['real'] = cspc.real
                         data['imag'] = cspc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
                     if self.data_dc is not None:
                         dc = self.data_dc
                         data = numpy.zeros( numpy.shape(dc), self.dtype )
                         data['real'] = dc.real
                         data['imag'] = dc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
             #         self.data_spc.fill(0)
             #
             #         if self.data_dc is not None:
             #             self.data_dc.fill(0)
             #
             #         if self.data_cspc is not None:
             #             self.data_cspc.fill(0)
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nWriteBlocks += 1
                     self.blockIndex += 1
             #         print "[Writing] Block = %d04" %self.blockIndex
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagDiscontinuousBlock:
                         self.data_spc.fill(0)
                         if self.dataOut.data_cspc is not None:
                             self.data_cspc.fill(0)
                         if self.dataOut.data_dc is not None:
                             self.data_dc.fill(0)
                         self.setNextFile()
                     if self.flagIsNewFile == 0:
                         self.setBasicHeader()
                     self.data_spc = self.dataOut.data_spc.copy()
                     if self.dataOut.data_cspc is not None:
                         self.data_cspc = self.dataOut.data_cspc.copy()
                     if self.dataOut.data_dc is not None:
                         self.data_dc = self.dataOut.data_dc.copy()
                     # #self.processingHeaderObj.dataBlocksPerFile)
                     if self.hasAllDataInBuffer():
             #            self.setFirstHeader()
                         self.writeNextBlock()
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
                     '''
                     dtype_width = self.getDtypeWidth()
                     pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
                     pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
                     blocksize = (pts2write_SelfSpectra*dtype_width)
                     if self.dataOut.data_cspc is not None:
                         pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
                         blocksize += (pts2write_CrossSpectra*dtype_width*2)
                     if self.dataOut.data_dc is not None:
                         pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
                         blocksize += (pts2write_DCchannels*dtype_width*2)
             #         blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.processingHeaderObj.dtype = 1 # Spectra
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
                     self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
                     self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
                     self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
                     if self.processingHeaderObj.totalSpectra > 0:
                         channelList = []
                         for channel in range(self.dataOut.nChannels):
                             channelList.append(channel)
                             channelList.append(channel)
                         pairsList = []
                         if self.dataOut.nPairs > 0:
                             for pair in self.dataOut.pairsList:
                                 pairsList.append(pair[0])
                                 pairsList.append(pair[1])
                         spectraComb = channelList + pairsList
                         spectraComb = numpy.array(spectraComb, dtype="u1")
                         self.processingHeaderObj.spectraComb = spectraComb
                     if self.dataOut.code is not None:
                         self.processingHeaderObj.code = self.dataOut.code
                         self.processingHeaderObj.nCode = self.dataOut.nCode
                         self.processingHeaderObj.nBaud = self.dataOut.nBaud
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                     self.processingHeaderObj.processFlags = self.getProcessFlags()
                     self.setBasicHeader()

schainpy/model/io/jroIO_usrp.py +3 -1

             '''
             Created on Jul 3, 2014
             @author: roj-idl71
             '''
             import os
             import datetime
             import numpy
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
-            from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
+            from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             try:
                 import digital_rf_hdf5
             except:
                 pass
             class USRPReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 def __init__(self, **kwargs):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self, **kwargs)
                     self.dataOut = Voltage()
                     self.__printInfo = True
                     self.__flagDiscontinuousBlock = False
                     self.__bufferIndex = 9999999
                     self.__ippKm = None
                     self.__codeType = 0
                     self.__nCode = None
                     self.__nBaud = None
                     self.__code = None
                 def __getCurrentSecond(self):
                     return self.__thisUnixSample/self.__sample_rate
                 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
                 def __setFileHeader(self):
                     '''
                     In this method will be initialized every parameter of dataOut object (header, no data)
                     '''
                     ippSeconds = 1.0*self.__nSamples/self.__sample_rate
                     nProfiles = 1.0/ippSeconds  #Number of profiles in one second
                     self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
                                                                                   txA=0,
                                                                                   txB=0,
                                                                                   nWindows=1,
                                                                                   nHeights=self.__nSamples,
                                                                                   firstHeight=self.__firstHeigth,
                                                                                   deltaHeight=self.__deltaHeigth,
                                                                                   codeType=self.__codeType,
                                                                                   nCode=self.__nCode, nBaud=self.__nBaud,
                                                                                   code = self.__code)
                     self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                 nProfiles=nProfiles,
                                                                 nChannels=len(self.__channelList),
                                                                 adcResolution=14)
                     self.dataOut.type = "Voltage"
                     self.dataOut.data = None
                     self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
             #        self.dataOut.nChannels = 0
             #        self.dataOut.nHeights = 0
                     self.dataOut.nProfiles = nProfiles
                     self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
                     self.dataOut.channelList = self.__channelList
                     self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
             #        self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData = True
                     #Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime = None
                     self.dataOut.timeZone = self.__timezone/60  #timezone like jroheader, difference in minutes between UTC and localtime
                     self.dataOut.dstFlag = 0
                     self.dataOut.errorCount = 0
                     self.dataOut.nCohInt = 1
                     self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
                     self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.ippSeconds = ippSeconds
                     #Time interval between profiles
                     #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.__frequency
                     self.dataOut.realtime = self.__online
                 def findDatafiles(self, path, startDate=None, endDate=None):
                     if not os.path.isdir(path):
                         return []
                     try:
                         digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
                     except:
                         digitalReadObj = digital_rf_hdf5.read_hdf5(path)
                     channelNameList = digitalReadObj.get_channels()
                     if not channelNameList:
                         return []
                     metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
                     sample_rate = metadata_dict['sample_rate'][0]
                     this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
                     try:
                         timezone = this_metadata_file['timezone'].value
                     except:
                         timezone = 0
                     startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(channelNameList[0])/sample_rate - timezone
                     startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
                     endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
                     if not startDate:
                         startDate = startDatetime.date()
                     if not endDate:
                         endDate = endDatatime.date()
                     dateList = []
                     thisDatetime = startDatetime
                     while(thisDatetime<=endDatatime):
                         thisDate = thisDatetime.date()
                         if thisDate < startDate:
                             continue
                         if thisDate > endDate:
                             break
                         dateList.append(thisDate)
                         thisDatetime += datetime.timedelta(1)
                     return dateList
                 def setup(self, path = None,
                                 startDate = None,
                                 endDate = None,
                                 startTime = datetime.time(0,0,0),
                                 endTime = datetime.time(23,59,59),
                                 channelList = None,
                                 nSamples = None,
                                 ippKm = 60,
                                 online = False,
                                 delay = 60,
                                 buffer_size = 1024,
                                 **kwargs):
                     '''
                     In this method we should set all initial parameters.
                     Inputs:
                         path
                         startDate
                         endDate
                         startTime
                         endTime
                         set
                         expLabel
                         ext
                         online
                         delay
                     '''
                     if not os.path.isdir(path):
                         raise ValueError("[Reading] Directory %s does not exist" %path)
                     try:
                         self.digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
                     except:
                         self.digitalReadObj = digital_rf_hdf5.read_hdf5(path)
                     channelNameList = self.digitalReadObj.get_channels()
                     if not channelNameList:
                         raise ValueError("[Reading] Directory %s does not have any files" %path)
                     if not channelList:
                         channelList = list(range(len(channelNameList)))
                     ##########  Reading metadata ######################
                     metadata_dict = self.digitalReadObj.get_rf_file_metadata(channelNameList[channelList[0]])
                     self.__sample_rate = metadata_dict['sample_rate'][0]
             #         self.__samples_per_file = metadata_dict['samples_per_file'][0]
                     self.__deltaHeigth = 1e6*0.15/self.__sample_rate
                     this_metadata_file = self.digitalReadObj.get_metadata(channelNameList[channelList[0]])
                     self.__frequency = None
                     try:
                         self.__frequency = this_metadata_file['center_frequencies'].value
                     except:
                         self.__frequency = this_metadata_file['fc'].value
                     if not self.__frequency:
                         raise ValueError("Center Frequency is not defined in metadata file")
                     try:
                         self.__timezone = this_metadata_file['timezone'].value
                     except:
                         self.__timezone = 0
                     self.__firstHeigth = 0
                     try:
                         codeType = this_metadata_file['codeType'].value
                     except:
                         codeType = 0
                     nCode = 1
                     nBaud = 1
                     code = numpy.ones((nCode, nBaud), dtype=numpy.int)
                     if codeType:
                         nCode = this_metadata_file['nCode'].value
                         nBaud = this_metadata_file['nBaud'].value
                         code = this_metadata_file['code'].value
                     if not ippKm:
                         try:
                             #seconds to km
                             ippKm = 1e6*0.15*this_metadata_file['ipp'].value
                         except:
                             ippKm = None
                     ####################################################
                     startUTCSecond = None
                     endUTCSecond = None
                     if startDate:
                         startDatetime = datetime.datetime.combine(startDate, startTime)
                         startUTCSecond = (startDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
                     if endDate:
                         endDatetime = datetime.datetime.combine(endDate, endTime)
                         endUTCSecond = (endDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
                     start_index, end_index = self.digitalReadObj.get_bounds(channelNameList[channelList[0]])
                     if not startUTCSecond:
                         startUTCSecond = start_index/self.__sample_rate
                     if start_index > startUTCSecond*self.__sample_rate:
                         startUTCSecond = start_index/self.__sample_rate
                     if not endUTCSecond:
                         endUTCSecond = end_index/self.__sample_rate
                     if end_index < endUTCSecond*self.__sample_rate:
                         endUTCSecond = end_index/self.__sample_rate
                     if not nSamples:
                         if not ippKm:
                             raise ValueError("[Reading] nSamples or ippKm should be defined")
                         nSamples = int(ippKm / (1e6*0.15/self.__sample_rate))
                     channelBoundList = []
                     channelNameListFiltered = []
                     for thisIndexChannel in channelList:
                         thisChannelName =  channelNameList[thisIndexChannel]
                         start_index, end_index = self.digitalReadObj.get_bounds(thisChannelName)
                         channelBoundList.append((start_index, end_index))
                         channelNameListFiltered.append(thisChannelName)
                     self.profileIndex = 0
                     self.__delay = delay
                     self.__ippKm = ippKm
                     self.__codeType = codeType
                     self.__nCode = nCode
                     self.__nBaud = nBaud
                     self.__code = code
                     self.__datapath = path
                     self.__online = online
                     self.__channelList = channelList
                     self.__channelNameList = channelNameListFiltered
                     self.__channelBoundList = channelBoundList
                     self.__nSamples = nSamples
                     self.__samples_to_read = int(buffer_size*nSamples)
                     self.__nChannels = len(self.__channelList)
                     self.__startUTCSecond = startUTCSecond
                     self.__endUTCSecond = endUTCSecond
                     self.__timeInterval = 1.0 * self.__samples_to_read/self.__sample_rate #Time interval
                     if online:
             #             self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
                         startUTCSecond = numpy.floor(endUTCSecond)
                     self.__thisUnixSample = int(startUTCSecond*self.__sample_rate) - self.__samples_to_read
                     self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype = numpy.complex)
                     self.__setFileHeader()
                     self.isConfig = True
                     print("[Reading] USRP Data was found from %s to %s " %(
                                                                   datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
                                                                   datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
                                                                   ))
                     print("[Reading] Starting process from %s to %s" %(datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
                                                                        datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
                                                                        ))
                 def __reload(self):
                     if not self.__online:
                         return
             #         print
             #         print "%s not in range [%s, %s]" %(
             #                                           datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
             #                                           datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
             #                                           datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
             #                                           )
                     print("[Reading] reloading metadata ...")
                     try:
                         self.digitalReadObj.reload(complete_update=True)
                     except:
                         self.digitalReadObj.reload()
                     start_index, end_index = self.digitalReadObj.get_bounds(self.__channelNameList[self.__channelList[0]])
                     if start_index > self.__startUTCSecond*self.__sample_rate:
                         self.__startUTCSecond = 1.0*start_index/self.__sample_rate
                     if end_index > self.__endUTCSecond*self.__sample_rate:
                         self.__endUTCSecond = 1.0*end_index/self.__sample_rate
                         print()
                         print("[Reading] New timerange found [%s, %s] " %(
                                                                   datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
                                                                   datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
                                                                   ))
                         return True
                     return False
                 def __readNextBlock(self, seconds=30, volt_scale = 218776):
                     '''
                     '''
                     #Set the next data
                     self.__flagDiscontinuousBlock = False
                     self.__thisUnixSample += self.__samples_to_read
                     if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
                         print("[Reading] There are no more data into selected time-range")
                         self.__reload()
                         if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
                             self.__thisUnixSample -=  self.__samples_to_read
                             return False
                     indexChannel = 0
                     dataOk = False
                     for thisChannelName in self.__channelNameList:
                         try:
                             result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
                                                                           self.__samples_to_read,
                                                                           thisChannelName)
                         except IOError as e:
                             #read next profile
                             self.__flagDiscontinuousBlock = True
                             print("[Reading] %s" %datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e)
                             break
                         if result.shape[0] != self.__samples_to_read:
                             self.__flagDiscontinuousBlock = True
                             print("[Reading] %s: Too few samples were found, just %d/%d  samples" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                                             result.shape[0],
                                                                                             self.__samples_to_read))
                             break
                         self.__data_buffer[indexChannel,:] = result*volt_scale
                         indexChannel += 1
                         dataOk = True
                     self.__utctime = self.__thisUnixSample/self.__sample_rate
                     if not dataOk:
                         return False
                     print("[Reading] %s: %d samples <> %f sec" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                      self.__samples_to_read,
                                                                      self.__timeInterval))
                     self.__bufferIndex = 0
                     return True
                 def __isBufferEmpty(self):
                     if self.__bufferIndex <= self.__samples_to_read - self.__nSamples:
                         return False
                     return True
                 def getData(self, seconds=30, nTries=5):
                     '''
                     This method gets the data from files and put the data into the dataOut object
                     In addition, increase el the buffer counter in one.
                     Return:
                         data    :    retorna un perfil de voltages (alturas * canales) copiados desde el
                                      buffer. Si no hay mas archivos a leer retorna None.
                     Affected:
                         self.dataOut
                         self.profileIndex
                         self.flagDiscontinuousBlock
                         self.flagIsNewBlock
                     '''
                     err_counter = 0
                     self.dataOut.flagNoData = True
                     if self.__isBufferEmpty():
                         self.__flagDiscontinuousBlock = False
                         while True:
                             if self.__readNextBlock():
                                 break
                             if self.__thisUnixSample > self.__endUTCSecond*self.__sample_rate:
                                 return False
                             if self.__flagDiscontinuousBlock:
                                 print('[Reading] discontinuous block found ... continue with the next block')
                                 continue
                             if not self.__online:
                                 return False
                             err_counter += 1
                             if err_counter > nTries:
                                 return False
                             print('[Reading] waiting %d seconds to read a new block' %seconds)
                             sleep(seconds)
                     self.dataOut.data = self.__data_buffer[:,self.__bufferIndex:self.__bufferIndex+self.__nSamples]
                     self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex)/self.__sample_rate
                     self.dataOut.flagNoData = False
                     self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
                     self.dataOut.profileIndex = self.profileIndex
                     self.__bufferIndex += self.__nSamples
                     self.profileIndex += 1
                     if self.profileIndex == self.dataOut.nProfiles:
                         self.profileIndex = 0
                     return True
                 def printInfo(self):
                     '''
                     '''
                     if self.__printInfo == False:
                         return
             #         self.systemHeaderObj.printInfo()
             #         self.radarControllerHeaderObj.printInfo()
                     self.__printInfo = False
                 def printNumberOfBlock(self):
                     '''
                     '''
                     print(self.profileIndex)
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                     self.getData(seconds=self.__delay)
                     return
+            @MPDecorator
             class USRPWriter(Operation):
                 '''
                 classdocs
                 '''
                 def __init__(self, **kwargs):
                     '''
                     Constructor
                     '''
                     Operation.__init__(self, **kwargs)
                     self.dataOut = None
                 def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
                     '''
                     In this method we should set all initial parameters.
                     Input:
                         dataIn        :        Input data will also be outputa data
                     '''
                     self.dataOut = dataIn
                     self.isConfig = True
                     return
                 def run(self, dataIn, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     Inputs:
                         dataIn        :        object with the data
                     '''
                     if not self.isConfig:
                         self.setup(dataIn, **kwargs)
             if __name__ == '__main__':
                 readObj = USRPReader()
                 while True:
                     readObj.run(path='/Volumes/DATA/haystack/passive_radar/')
             #         readObj.printInfo()
                     readObj.printNumberOfBlock()

schainpy/model/io/jroIO_voltage.py +1 -5

             '''
             Created on Jul 2, 2014
             @author: roj-idl71
             '''
             import numpy
             from .jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jrodata import Voltage
-            import zmq
-            import tempfile
-            from io import StringIO
-            # from _sha import blocksize
-            @MPDecorator
             class VoltageReader(JRODataReader, ProcessingUnit):
                 """
                 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
                 perfiles*alturas*canales) son almacenados en la variable "buffer".
                                          perfiles * alturas * canales
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = VoltageReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         #to get one profile
                         profile =  readerObj.getData()
                         #print the profile
                         print profile
                         #If you want to see all datablock
                         print readerObj.datablock
                         if readerObj.flagNoMoreFiles:
                             break
                 """
                 def __init__(self):
                     """
                     Inicializador de la clase VoltageReader para la lectura de datos de voltage.
                     Input:
                         dataOut    :    Objeto de la clase Voltage. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Variables afectadas:
                         self.dataOut
                     Return:
                         None
                     """
                     ProcessingUnit.__init__(self)
                     self.ext = ".r"
                     self.optchar = "D"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.lastUTTime = 0
                     self.profileIndex = 2**32 - 1
                     self.dataOut = Voltage()
                     self.selBlocksize = None
                     self.selBlocktime = None
                 def createObjByDefault(self):
                     dataObj = Voltage()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock * self.nTxs:
                         return 1
                     return 0
                 def getBlockDimension(self):
                     """
                         Obtiene la cantidad de puntos a leer por cada bloque de datos
                         Affected:
                             self.blocksize
                         Return:
                             None
                     """
                     pts2read = self.processingHeaderObj.profilesPerBlock * \
                         self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
                     self.blocksize = pts2read
                 def readBlock(self):
                     """
                         readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
                         (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                         (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                         es seteado a 0
                         Inputs:
                             None
                         Return:
                             None
                         Affected:
                             self.profileIndex
                             self.datablock
                             self.flagIsNewFile
                             self.flagIsNewBlock
                             self.nTotalBlocks
                         Exceptions:
                             Si un bloque leido no es un bloque valido
                     """
                     # if self.server is not None:
                     #     self.zBlock = self.receiver.recv()
                     #     self.zHeader = self.zBlock[:24]
                     #     self.zDataBlock = self.zBlock[24:]
                     #     junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
                     #     self.processingHeaderObj.profilesPerBlock = 240
                     #     self.processingHeaderObj.nHeights = 248
                     #     self.systemHeaderObj.nChannels
                     # else:
                     current_pointer_location = self.fp.tell()
                     junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
                     try:
                         junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
                                              self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
                     except:
                         # print "The read block (%3d) has not enough data" %self.nReadBlocks
                         if self.waitDataBlock(pointer_location=current_pointer_location):
                             junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
                             junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
                                                  self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
                     #             return 0
                     # Dimensions : nChannels, nProfiles, nSamples
                     junk = numpy.transpose(junk, (2, 0, 1))
                     self.datablock = junk['real'] + junk['imag'] * 1j
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.getBasicHeader()
                     self.dataOut.processingHeaderObj = self.processingHeaderObj.copy()
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
                     if self.nTxs > 1:
                         self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
                     # Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
                     #         self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
                     #
                     #         if self.radarControllerHeaderObj.code is not None:
                     #
                     #             self.dataOut.nCode = self.radarControllerHeaderObj.nCode
                     #
                     #             self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
                     #
                     #             self.dataOut.code = self.radarControllerHeaderObj.code
                     self.dataOut.dtype = self.dtype
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.heightList = numpy.arange(
                         self.processingHeaderObj.nHeights) * self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
                     self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     # asumo q la data no esta decodificada
                     self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode
                     # asumo q la data no esta sin flip
                     self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip
                     self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
                 def reshapeData(self):
                     if self.nTxs < 0:
                         return
                     if self.nTxs == 1:
                         return
                     if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1. / self.nTxs) != 0:
                         raise ValueError("1./nTxs (=%f), should be a multiple of nProfiles (=%d)" % (
 . / self.nTxs, self.processingHeaderObj.profilesPerBlock))
                     if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
                         raise ValueError("nTxs (=%d), should be a multiple of nHeights (=%d)" % (
                             self.nTxs, self.processingHeaderObj.nHeights))
                     self.datablock = self.datablock.reshape(
                         (self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock * self.nTxs, int(self.processingHeaderObj.nHeights / self.nTxs)))
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock * self.nTxs
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights / self.nTxs) * \
                         self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
                     self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
                     return
                 def readFirstHeaderFromServer(self):
                     self.getFirstHeader()
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
                                                PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
                     else:
                         raise ValueError('Data type was not defined')
                     self.dtype = datatype_str
                     #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
                         self.firstHeaderSize + self.basicHeaderSize * \
                         (self.processingHeaderObj.dataBlocksPerFile - 1)
                     #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
                     #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def getFromServer(self):
                     self.flagDiscontinuousBlock = 0
                     self.profileIndex = 0
                     self.flagIsNewBlock = 1
                     self.dataOut.flagNoData = False
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     self.blockPointer = 0
                     block = self.receiver.recv()
                     self.basicHeaderObj.read(block[self.blockPointer:])
                     self.blockPointer += self.basicHeaderObj.length
                     self.systemHeaderObj.read(block[self.blockPointer:])
                     self.blockPointer += self.systemHeaderObj.length
                     self.radarControllerHeaderObj.read(block[self.blockPointer:])
                     self.blockPointer += self.radarControllerHeaderObj.length
                     self.processingHeaderObj.read(block[self.blockPointer:])
                     self.blockPointer += self.processingHeaderObj.length
                     self.readFirstHeaderFromServer()
                     timestamp = self.basicHeaderObj.get_datatime()
                     print('[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp))
                     current_pointer_location = self.blockPointer
                     junk = numpy.fromstring(
                         block[self.blockPointer:], self.dtype, self.blocksize)
                     try:
                         junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
                                              self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
                     except:
                         # print "The read block (%3d) has not enough data" %self.nReadBlocks
                         if self.waitDataBlock(pointer_location=current_pointer_location):
                             junk = numpy.fromstring(
                                 block[self.blockPointer:], self.dtype, self.blocksize)
                             junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
                                                  self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
                     #               return 0
                     # Dimensions : nChannels, nProfiles, nSamples
                     junk = numpy.transpose(junk, (2, 0, 1))
                     self.datablock = junk['real'] + junk['imag'] * 1j
                     self.profileIndex = 0
                     if self.selBlocksize == None:
                         self.selBlocksize = self.dataOut.nProfiles
                     if self.selBlocktime != None:
                         if self.dataOut.nCohInt is not None:
                             nCohInt = self.dataOut.nCohInt
                         else:
                             nCohInt = 1
                         self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
                             nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
                     self.dataOut.data = self.datablock[:,
                                                        self.profileIndex:self.profileIndex + self.selBlocksize, :]
                     datasize = self.dataOut.data.shape[1]
                     if datasize < self.selBlocksize:
                         buffer = numpy.zeros(
                             (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
                         buffer[:, :datasize, :] = self.dataOut.data
                         self.dataOut.data = buffer
                         self.profileIndex = blockIndex
                     self.dataOut.flagDataAsBlock = True
                     self.flagIsNewBlock = 1
                     self.dataOut.realtime = self.online
                     return self.dataOut.data
                 def getData(self):
                     """
                         getData obtiene una unidad de datos del buffer de lectura, un perfil,  y la copia al objeto self.dataOut
                         del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
                         en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
                         "readNextBlock"
                         Ademas incrementa el contador del buffer "self.profileIndex" en 1.
                         Return:
                             Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
                             es igual al total de perfiles leidos desde el archivo.
                             Si self.getByBlock == False:
                                 self.dataOut.data = buffer[:, thisProfile, :]
                                 shape = [nChannels, nHeis]
                             Si self.getByBlock == True:
                                 self.dataOut.data = buffer[:, :, :]
                                 shape = [nChannels, nProfiles, nHeis]
                         Variables afectadas:
                             self.dataOut
                             self.profileIndex
                         Affected:
                             self.dataOut
                             self.profileIndex
                             self.flagDiscontinuousBlock
                             self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         return 0
                     self.flagDiscontinuousBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not(self.readNextBlock()):
                             return 0
                         self.getFirstHeader()
                         self.reshapeData()
                     if self.datablock is None:
                         self.dataOut.flagNoData = True
                         return 0
                     if not self.getByBlock:
                         """
                             Return profile by profile
                             If nTxs > 1 then one profile is divided by nTxs and number of total
                             blocks is increased by nTxs (nProfiles *= nTxs)
                         """
                         self.dataOut.flagDataAsBlock = False
                         self.dataOut.data = self.datablock[:, self.profileIndex, :]
                         self.dataOut.profileIndex = self.profileIndex
                         self.profileIndex += 1
                     else:
                         """
                             Return a block
                         """
                         if self.selBlocksize == None:
                             self.selBlocksize = self.dataOut.nProfiles
                         if self.selBlocktime != None:
                             if self.dataOut.nCohInt is not None:
                                 nCohInt = self.dataOut.nCohInt
                             else:
                                 nCohInt = 1
                             self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
                                 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
                         self.dataOut.data = self.datablock[:,
                                                            self.profileIndex:self.profileIndex + self.selBlocksize, :]
                         self.profileIndex += self.selBlocksize
                         datasize = self.dataOut.data.shape[1]
                         if datasize < self.selBlocksize:
                             buffer = numpy.zeros(
                                 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
                             buffer[:, :datasize, :] = self.dataOut.data
                             while datasize < self.selBlocksize:  # Not enough profiles to fill the block
                                 if not(self.readNextBlock()):
                                     return 0
                                 self.getFirstHeader()
                                 self.reshapeData()
                                 if self.datablock is None:
                                     self.dataOut.flagNoData = True
                                     return 0
                                 # stack data
                                 blockIndex = self.selBlocksize - datasize
                                 datablock1 = self.datablock[:, :blockIndex, :]
                                 buffer[:, datasize:datasize +
                                        datablock1.shape[1], :] = datablock1
                                 datasize += datablock1.shape[1]
                             self.dataOut.data = buffer
                             self.profileIndex = blockIndex
                         self.dataOut.flagDataAsBlock = True
                         self.dataOut.nProfiles = self.dataOut.data.shape[1]
                     self.dataOut.flagNoData = False
                     self.getBasicHeader()
                     self.dataOut.realtime = self.online
                     return self.dataOut.data
             @MPDecorator
             class VoltageWriter(JRODataWriter, Operation):
                 """
                 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 ext = ".r"
                 optchar = "D"
                 shapeBuffer = None
                 def __init__(self):#, **kwargs):
                     """
                     Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                     Return: None
                     """
                     Operation.__init__(self)#, **kwargs)
                     self.nTotalBlocks = 0
                     self.profileIndex = 0
                     self.isConfig = False
                     self.fp = None
                     self.flagIsNewFile = 1
                     self.blockIndex = 0
                     self.flagIsNewBlock = 0
                     self.setFile = None
                     self.dtype = None
                     self.path = None
                     self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
                         return 1
                     return 0
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
                                         self.processingHeaderObj.nHeights,
                                         self.systemHeaderObj.nChannels)
                     self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
                                                   self.processingHeaderObj.profilesPerBlock,
                                                   self.processingHeaderObj.nHeights),
                                                  dtype=numpy.dtype('complex64'))
                 def writeBlock(self):
                     """
                     Escribe el buffer en el file designado
                     Affected:
                         self.profileIndex
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.blockIndex
                     Return: None
                     """
                     data = numpy.zeros(self.shapeBuffer, self.dtype)
                     junk = numpy.transpose(self.datablock, (1, 2, 0))
                     data['real'] = junk.real
                     data['imag'] = junk.imag
                     data = data.reshape((-1))
                     data.tofile(self.fp)
                     self.datablock.fill(0)
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.blockIndex += 1
                     self.nTotalBlocks += 1
             #         print "[Writing] Block = %04d" %self.blockIndex
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.flagIsNewBlock
                         self.profileIndex
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagDiscontinuousBlock:
                         self.datablock.fill(0)
                         self.profileIndex = 0
                         self.setNextFile()
                     if self.profileIndex == 0:
                         self.setBasicHeader()
                     self.datablock[:, self.profileIndex, :] = self.dataOut.data
                     self.profileIndex += 1
                     if self.hasAllDataInBuffer():
                         # if self.flagIsNewFile:
                         self.writeNextBlock()
             #            self.setFirstHeader()
                     return 1
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
                     '''
                     dtype_width = self.getDtypeWidth()
                     blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels *
                                     self.profilesPerBlock * dtype_width * 2)
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.processingHeaderObj.dtype = 0  # Voltage
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     # podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.nWindows = 1
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
                     # Cuando la data de origen es de tipo Voltage
                     self.processingHeaderObj.nIncohInt = 1
                     # Cuando la data de origen es de tipo Voltage
                     self.processingHeaderObj.totalSpectra = 0
                     if self.dataOut.code is not None:
                         self.processingHeaderObj.code = self.dataOut.code
                         self.processingHeaderObj.nCode = self.dataOut.nCode
                         self.processingHeaderObj.nBaud = self.dataOut.nBaud
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - \
                             self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                     self.processingHeaderObj.processFlags = self.getProcessFlags()
                     self.setBasicHeader()
   No newline at end of file

schainpy/model/proc/bltrproc_parameters.py +2 -2

             '''
             Created on Oct 24, 2016
             @author: roj- LouVD
             '''
             import numpy
             import copy
             import datetime
             import time
             from time import gmtime
             from numpy import transpose
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jrodata import Parameters
-            @MPDecorator
             class BLTRParametersProc(ProcessingUnit):
                 '''
                 Processing unit for BLTR parameters data (winds)
                 Inputs:
                     self.dataOut.nmodes - Number of operation modes
                     self.dataOut.nchannels - Number of channels
                     self.dataOut.nranges - Number of ranges
                     self.dataOut.data_SNR - SNR array
                     self.dataOut.data_output - Zonal, Vertical and Meridional velocity array
                     self.dataOut.height - Height array (km)
                     self.dataOut.time - Time array (seconds)
                     self.dataOut.fileIndex -Index of the file currently read
                     self.dataOut.lat - Latitude coordinate of BLTR location
                     self.dataOut.doy - Experiment doy (number of the day in the current year)
                     self.dataOut.month - Experiment month
                     self.dataOut.day - Experiment day
                     self.dataOut.year - Experiment year
                 '''
                 def __init__(self):
                     '''
                     Inputs: None
                     '''
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                 def setup(self, mode):
                     '''
                     '''
                     self.dataOut.mode = mode
                 def run(self, mode, snr_threshold=None):
                     '''
                     Inputs:
                         mode = High resolution (0) or Low resolution (1) data
                         snr_threshold = snr filter value
                     '''
                     if not self.isConfig:
                         self.setup(mode)
                         self.isConfig = True
                     if self.dataIn.type == 'Parameters':
                         self.dataOut.copy(self.dataIn)
                     self.dataOut.data_param = self.dataOut.data[mode]
                     self.dataOut.heightList = self.dataOut.height[0]
                     self.dataOut.data_SNR = self.dataOut.data_SNR[mode]
                     if snr_threshold is not None:
                         SNRavg = numpy.average(self.dataOut.data_SNR, axis=0)
                         SNRavgdB = 10*numpy.log10(SNRavg)
                         for i in range(3):
                             self.dataOut.data_param[i][SNRavgdB <= snr_threshold] = numpy.nan
             # TODO
-            @MPDecorator
             class OutliersFilter(Operation):
                 def __init__(self):
                     '''
                     '''
                     Operation.__init__(self)
                 def run(self, svalue2, method, factor, filter, npoints=9):
                     '''
                     Inputs:
                         svalue    -  string to select array velocity
                         svalue2    -  string to choose axis filtering
                         method    - 0 for SMOOTH or 1 for MEDIAN
                         factor    - number used to set threshold
                         filter    - 1 for data filtering using the standard deviation criteria else 0
                         npoints    - number of points for mask filter
                     '''
                     print('    Outliers Filter {} {} / threshold = {}'.format(svalue, svalue, factor))
                     yaxis = self.dataOut.heightList
                     xaxis = numpy.array([[self.dataOut.utctime]])
                     # Zonal
                     value_temp = self.dataOut.data_output[0]
                     # Zonal
                     value_temp = self.dataOut.data_output[1]
                     # Vertical
                     value_temp = numpy.transpose(self.dataOut.data_output[2])
                     htemp = yaxis
                     std = value_temp
                     for h in range(len(htemp)):
                         nvalues_valid = len(numpy.where(numpy.isfinite(value_temp[h]))[0])
                         minvalid = npoints
                         #only if valid values greater than the minimum required (10%)
                         if nvalues_valid > minvalid:
                             if method == 0:
                                 #SMOOTH
                                 w = value_temp[h] - self.Smooth(input=value_temp[h], width=npoints, edge_truncate=1)
                             if method == 1:
                                 #MEDIAN
                                 w = value_temp[h] - self.Median(input=value_temp[h], width = npoints)
                             dw = numpy.std(w[numpy.where(numpy.isfinite(w))],ddof = 1)
                             threshold = dw*factor
                             value_temp[numpy.where(w > threshold),h] = numpy.nan
                             value_temp[numpy.where(w < -1*threshold),h] = numpy.nan
                     #At the end
                     if svalue2 == 'inHeight':
                         value_temp = numpy.transpose(value_temp)
                     output_array[:,m] = value_temp
                     if svalue == 'zonal':
                         self.dataOut.data_output[0] = output_array
                     elif svalue == 'meridional':
                         self.dataOut.data_output[1] = output_array
                     elif svalue == 'vertical':
                         self.dataOut.data_output[2] = output_array
                     return self.dataOut.data_output
                 def Median(self,input,width):
                     '''
                     Inputs:
                         input - Velocity array
                         width - Number of points for mask filter
                     '''
                     if numpy.mod(width,2) == 1:
                         pc = int((width - 1) / 2)
                     cont = 0
                     output = []
                     for i in range(len(input)):
                         if i >= pc and i < len(input) - pc:
                             new2 = input[i-pc:i+pc+1]
                             temp = numpy.where(numpy.isfinite(new2))
                             new = new2[temp]
                             value = numpy.median(new)
                             output.append(value)
                     output = numpy.array(output)
                     output = numpy.hstack((input[0:pc],output))
                     output = numpy.hstack((output,input[-pc:len(input)]))
                     return output
                 def Smooth(self,input,width,edge_truncate = None):
                     '''
                     Inputs:
                         input - Velocity array
                         width - Number of points for mask filter
                         edge_truncate - 1 for truncate the convolution product else
                     '''
                     if numpy.mod(width,2) == 0:
                         real_width = width + 1
                         nzeros = width / 2
                     else:
                         real_width = width
                         nzeros = (width - 1) / 2
                     half_width = int(real_width)/2
                     length = len(input)
                     gate = numpy.ones(real_width,dtype='float')
                     norm_of_gate = numpy.sum(gate)
                     nan_process = 0
                     nan_id = numpy.where(numpy.isnan(input))
                     if len(nan_id[0]) > 0:
                         nan_process = 1
                         pb = numpy.zeros(len(input))
                         pb[nan_id] = 1.
                         input[nan_id] = 0.
                     if edge_truncate == True:
                         output = numpy.convolve(input/norm_of_gate,gate,mode='same')
                     elif edge_truncate == False or edge_truncate == None:
                         output = numpy.convolve(input/norm_of_gate,gate,mode='valid')
                         output = numpy.hstack((input[0:half_width],output))
                         output = numpy.hstack((output,input[len(input)-half_width:len(input)]))
                     if nan_process:
                         pb = numpy.convolve(pb/norm_of_gate,gate,mode='valid')
                         pb = numpy.hstack((numpy.zeros(half_width),pb))
                         pb = numpy.hstack((pb,numpy.zeros(half_width)))
                         output[numpy.where(pb > 0.9999)] = numpy.nan
                         input[nan_id] = numpy.nan
                     return output
                 def Average(self,aver=0,nhaver=1):
                     '''
                     Inputs:
                         aver - Indicates the time period over which is averaged or consensus data
                         nhaver - Indicates the decimation factor in heights
                     '''
                     nhpoints = 48
                     lat_piura = -5.17
                     lat_huancayo = -12.04
                     lat_porcuya = -5.8
                     if '%2.2f'%self.dataOut.lat == '%2.2f'%lat_piura:
                         hcm = 3.
                         if self.dataOut.year == 2003 :
                             if self.dataOut.doy >= 25 and self.dataOut.doy < 64:
                                 nhpoints = 12
                     elif '%2.2f'%self.dataOut.lat == '%2.2f'%lat_huancayo:
                         hcm = 3.
                         if self.dataOut.year == 2003 :
                             if self.dataOut.doy >= 25 and self.dataOut.doy < 64:
                                 nhpoints = 12
                     elif '%2.2f'%self.dataOut.lat == '%2.2f'%lat_porcuya:
                         hcm = 5.#2
                     pdata = 0.2
                     taver = [1,2,3,4,6,8,12,24]
                     t0 = 0
                     tf = 24
                     ntime =(tf-t0)/taver[aver]
                     ti = numpy.arange(ntime)
                     tf = numpy.arange(ntime) + taver[aver]
                     old_height = self.dataOut.heightList
                     if nhaver > 1:
                         num_hei = len(self.dataOut.heightList)/nhaver/self.dataOut.nmodes
                         deltha = 0.05*nhaver
                         minhvalid = pdata*nhaver
                         for im in range(self.dataOut.nmodes):
                             new_height = numpy.arange(num_hei)*deltha + self.dataOut.height[im,0] + deltha/2.
                     data_fHeigths_List = []
                     data_fZonal_List = []
                     data_fMeridional_List = []
                     data_fVertical_List = []
                     startDTList = []
                     for i in range(ntime):
                         height = old_height
                         start = datetime.datetime(self.dataOut.year,self.dataOut.month,self.dataOut.day) + datetime.timedelta(hours = int(ti[i])) - datetime.timedelta(hours = 5)
                         stop = datetime.datetime(self.dataOut.year,self.dataOut.month,self.dataOut.day) + datetime.timedelta(hours = int(tf[i])) - datetime.timedelta(hours = 5)
                         limit_sec1 = time.mktime(start.timetuple())
                         limit_sec2 = time.mktime(stop.timetuple())
                         t1 = numpy.where(self.f_timesec >= limit_sec1)
                         t2 = numpy.where(self.f_timesec < limit_sec2)
                         time_select = []
                         for val_sec in t1[0]:
                             if val_sec in t2[0]:
                                 time_select.append(val_sec)
                         time_select = numpy.array(time_select,dtype = 'int')
                         minvalid = numpy.ceil(pdata*nhpoints)
                         zon_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan
                         mer_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan
                         ver_aver = numpy.zeros([self.dataOut.nranges,self.dataOut.nmodes],dtype='f4') + numpy.nan
                         if nhaver > 1:
                             new_zon_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan
                             new_mer_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan
                             new_ver_aver = numpy.zeros([num_hei,self.dataOut.nmodes],dtype='f4') + numpy.nan
                         if len(time_select) > minvalid:
                             time_average = self.f_timesec[time_select]
                             for im in range(self.dataOut.nmodes):
                                 for ih in range(self.dataOut.nranges):
                                     if numpy.sum(numpy.isfinite(self.f_zon[time_select,ih,im])) >= minvalid:
                                         zon_aver[ih,im] = numpy.nansum(self.f_zon[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_zon[time_select,ih,im]))
                                     if numpy.sum(numpy.isfinite(self.f_mer[time_select,ih,im])) >= minvalid:
                                         mer_aver[ih,im] = numpy.nansum(self.f_mer[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_mer[time_select,ih,im]))
                                     if numpy.sum(numpy.isfinite(self.f_ver[time_select,ih,im])) >= minvalid:
                                         ver_aver[ih,im] = numpy.nansum(self.f_ver[time_select,ih,im]) / numpy.sum(numpy.isfinite(self.f_ver[time_select,ih,im]))
                                 if nhaver > 1:
                                     for ih in range(num_hei):
                                         hvalid = numpy.arange(nhaver) + nhaver*ih
                                         if numpy.sum(numpy.isfinite(zon_aver[hvalid,im])) >= minvalid:
                                             new_zon_aver[ih,im] = numpy.nansum(zon_aver[hvalid,im]) /  numpy.sum(numpy.isfinite(zon_aver[hvalid,im]))
                                         if numpy.sum(numpy.isfinite(mer_aver[hvalid,im])) >= minvalid:
                                             new_mer_aver[ih,im] = numpy.nansum(mer_aver[hvalid,im]) /  numpy.sum(numpy.isfinite(mer_aver[hvalid,im]))
                                         if numpy.sum(numpy.isfinite(ver_aver[hvalid,im])) >= minvalid:
                                             new_ver_aver[ih,im] = numpy.nansum(ver_aver[hvalid,im]) /  numpy.sum(numpy.isfinite(ver_aver[hvalid,im]))
                             if nhaver > 1:
                                 zon_aver = new_zon_aver
                                 mer_aver = new_mer_aver
                                 ver_aver = new_ver_aver
                                 height = new_height
                             tstart = time_average[0]
                             tend = time_average[-1]
                             startTime = time.gmtime(tstart)
                             year = startTime.tm_year
                             month = startTime.tm_mon
                             day = startTime.tm_mday
                             hour = startTime.tm_hour
                             minute = startTime.tm_min
                             second = startTime.tm_sec
                             startDTList.append(datetime.datetime(year,month,day,hour,minute,second))
                             o_height = numpy.array([])
                             o_zon_aver = numpy.array([])
                             o_mer_aver = numpy.array([])
                             o_ver_aver = numpy.array([])
                             if self.dataOut.nmodes > 1:
                                 for im in range(self.dataOut.nmodes):
                                     if im == 0:
                                         h_select = numpy.where(numpy.bitwise_and(height[0,:] >=0,height[0,:] <= hcm,numpy.isfinite(height[0,:])))
                                     else:
                                         h_select = numpy.where(numpy.bitwise_and(height[1,:] > hcm,height[1,:] < 20,numpy.isfinite(height[1,:])))
                                     ht = h_select[0]
                                     o_height = numpy.hstack((o_height,height[im,ht]))
                                     o_zon_aver = numpy.hstack((o_zon_aver,zon_aver[ht,im]))
                                     o_mer_aver = numpy.hstack((o_mer_aver,mer_aver[ht,im]))
                                     o_ver_aver = numpy.hstack((o_ver_aver,ver_aver[ht,im]))
                                 data_fHeigths_List.append(o_height)
                                 data_fZonal_List.append(o_zon_aver)
                                 data_fMeridional_List.append(o_mer_aver)
                                 data_fVertical_List.append(o_ver_aver)
                             else:
                                 h_select = numpy.where(numpy.bitwise_and(height[0,:] <= hcm,numpy.isfinite(height[0,:])))
                                 ht = h_select[0]
                                 o_height = numpy.hstack((o_height,height[im,ht]))
                                 o_zon_aver = numpy.hstack((o_zon_aver,zon_aver[ht,im]))
                                 o_mer_aver = numpy.hstack((o_mer_aver,mer_aver[ht,im]))
                                 o_ver_aver = numpy.hstack((o_ver_aver,ver_aver[ht,im]))
                                 data_fHeigths_List.append(o_height)
                                 data_fZonal_List.append(o_zon_aver)
                                 data_fMeridional_List.append(o_mer_aver)
                                 data_fVertical_List.append(o_ver_aver)
                     return startDTList, data_fHeigths_List, data_fZonal_List, data_fMeridional_List, data_fVertical_List

schainpy/model/proc/jroproc_amisr.py +1 -1

             '''
             @author: Daniel Suarez
             '''
             import numpy
             from .jroproc_base import ProcessingUnit, Operation
             from schainpy.model.data.jroamisr import AMISR
             class AMISRProc(ProcessingUnit):
                 def __init__(self, **kwargs):
                     ProcessingUnit.__init__(self, **kwargs)
                     self.objectDict = {}
                     self.dataOut = AMISR()
                 def run(self):
                     if self.dataIn.type == 'AMISR':
                         self.dataOut.copy(self.dataIn)
-            class PrintInfo(Operation):
+            class PrintInfoAMISR(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.__isPrinted = False
                 def run(self, dataOut):
                     if not self.__isPrinted:
                         print('Number of Records by File: %d'%dataOut.nRecords)
                         print('Number of Pulses: %d'%dataOut.nProfiles)
                         print('Number of Pulses by Frame: %d'%dataOut.npulseByFrame)
                         print('Number of Samples by Pulse: %d'%len(dataOut.heightList))
                         print('Ipp Seconds: %f'%dataOut.ippSeconds)
                         print('Number of Beams: %d'%dataOut.nBeams)
                         print('BeamCodes:')
                         beamStrList = ['Beam %d -> Code=%d, azimuth=%2.2f,  zenith=%2.2f, gain=%2.2f'%(k,v[0],v[1],v[2],v[3]) for k,v in list(dataOut.beamCodeDict.items())]
                         for b in beamStrList:
                             print(b)
                         self.__isPrinted = True
                     return
             class BeamSelector(Operation):
                 profileIndex = None
                 nProfiles = None
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.profileIndex = 0
                     self.__isConfig = False
                 def incIndex(self):
                     self.profileIndex += 1
                     if self.profileIndex >= self.nProfiles:
                         self.profileIndex = 0
                 def isProfileInRange(self, minIndex, maxIndex):
                     if self.profileIndex < minIndex:
                         return False
                     if self.profileIndex > maxIndex:
                         return False
                     return True
                 def isProfileInList(self, profileList):
                     if self.profileIndex not in profileList:
                         return False
                     return True
                 def run(self, dataOut, beam=None):
                     dataOut.flagNoData = True
                     if not(self.__isConfig):
                         self.nProfiles = dataOut.nProfiles
                         self.profileIndex = dataOut.profileIndex
                         self.__isConfig = True
                     if beam != None:
                         if self.isProfileInList(dataOut.beamRangeDict[beam]):
                             beamInfo = dataOut.beamCodeDict[beam]
                             dataOut.azimuth = beamInfo[1]
                             dataOut.zenith = beamInfo[2]
                             dataOut.gain = beamInfo[3]
                             dataOut.flagNoData = False
                         self.incIndex()
                         return 1
                     else:
                         raise ValueError("BeamSelector needs beam value")
                     return 0
             class ProfileToChannels(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.__isConfig = False
                     self.__counter_chan = 0
                     self.buffer = None
                 def isProfileInList(self, profileList):
                     if self.profileIndex not in profileList:
                         return False
                     return True
                 def run(self, dataOut):
                     dataOut.flagNoData = True
                     if not(self.__isConfig):
                         nchannels = len(list(dataOut.beamRangeDict.keys()))
                         nsamples = dataOut.nHeights
                         self.buffer = numpy.zeros((nchannels, nsamples), dtype = 'complex128')
                         dataOut.beam.codeList = [dataOut.beamCodeDict[x][0] for x in range(nchannels)]
                         dataOut.beam.azimuthList = [dataOut.beamCodeDict[x][1] for x in range(nchannels)]
                         dataOut.beam.zenithList = [dataOut.beamCodeDict[x][2] for x in range(nchannels)]
                         self.__isConfig = True
                     for i in range(self.buffer.shape[0]):
                         if dataOut.profileIndex in dataOut.beamRangeDict[i]:
                             self.buffer[i,:] = dataOut.data
                             break
                     self.__counter_chan += 1
                     if self.__counter_chan >= self.buffer.shape[0]:
                         self.__counter_chan = 0
                         dataOut.data = self.buffer.copy()
                         dataOut.channelList = list(range(self.buffer.shape[0]))
                         self.__isConfig = False
                         dataOut.flagNoData = False
                     pass
   No newline at end of file

schainpy/model/proc/jroproc_base.py +53 -275

@@ -1,429 +1,207
1	'''	1	'''
2	Updated for multiprocessing	2	Base clases to create Processing units and operations, the MPDecorator
3	Author : Sergio Cortez	3	must be used in plotting and writing operations to allow to run as an
4	Jan 2018	4	external process.
5	Abstract:
6	Base class for processing units and operations. A decorator provides multiprocessing features and interconnect the processes created.
7	The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
8	The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
9
10	Based on:
11	$Author: murco $
12	$Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
13	'''	5	'''
14		6
15	import os
16	import sys
17	import inspect	7	import inspect
18	import zmq	8	import zmq
19	import time	9	import time
20	import pickle	10	import pickle
21	import traceback	11	import traceback
22	try:	12	try:
23	from queue import Queue	13	from queue import Queue
24	except:	14	except:
25	from Queue import Queue	15	from Queue import Queue
26	from threading import Thread	16	from threading import Thread
27	from multiprocessing import Process	17	from multiprocessing import Process, Queue
28
29	from schainpy.utils import log	18	from schainpy.utils import log
30		19
31		20
32	class ProcessingUnit(object):	21	class ProcessingUnit(object):
		22	'''
		23	Base class to create Signal Chain Units
		24	'''
33		25
34	"""
35	Update - Jan 2018 - MULTIPROCESSING
36	All the "call" methods present in the previous base were removed.
37	The majority of operations are independant processes, thus
38	the decorator is in charge of communicate the operation processes
39	with the proccessing unit via IPC.
40
41	The constructor does not receive any argument. The remaining methods
42	are related with the operations to execute.
43
44
45	"""
46	proc_type = 'processing'	26	proc_type = 'processing'
47	__attrs__ = []
48		27
49	def __init__(self):	28	def __init__(self):
50		29
51	self.dataIn = None	30	self.dataIn = None
52	self.dataOut = None	31	self.dataOut = None
53	self.isConfig = False	32	self.isConfig = False
54	self.operations = []	33	self.operations = []
55	self.plots = []	34
		35	def setInput(self, unit):
		36
		37	self.dataIn = unit.dataOut
56		38
57	def getAllowedArgs(self):	39	def getAllowedArgs(self):
58	if hasattr(self, '__attrs__'):	40	if hasattr(self, '__attrs__'):
59	return self.__attrs__	41	return self.__attrs__
60	else:	42	else:
61	return inspect.getargspec(self.run).args	43	return inspect.getargspec(self.run).args
62		44
63	def addOperation(self, conf, operation):	45	def addOperation(self, conf, operation):
64	~~"""~~	46	'''
65	This method is used in the controller, and update the dictionary containing the operations to execute. The dict	47	'''
66	posses the id of the operation process (IPC purposes)
67
68	Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
69	identificador asociado a este objeto.
70
71	Input:
72
73	object : objeto de la clase "Operation"
74
75	Return:
76
77	objId : identificador del objeto, necesario para comunicar con master(procUnit)
78	"""
79
80	self.operations.append(
81	(operation, conf.type, conf.id, conf.getKwargs()))
82		48
83	if 'plot' in self.name.lower():	49	self.operations.append((operation, conf.type, conf.getKwargs()))
84	self.plots.append(operation.CODE)
85		50
86	def getOperationObj(self, objId):	51	def getOperationObj(self, objId):
87		52
88	if objId not in list(self.operations.keys()):	53	if objId not in list(self.operations.keys()):
89	return None	54	return None
90		55
91	return self.operations[objId]	56	return self.operations[objId]
92		57
93	def ~~operation~~(self, **kwargs):	58	def call(self, **kwargs):
94	~~"""~~	59	'''
95	Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los	60	'''
96	atributos del objeto dataOut
97		61
98	~~Inpu~~t:	62	try:
		63	if self.dataIn is not None and self.dataIn.flagNoData and not self.dataIn.error:
		64	return self.dataIn.isReady()
		65	elif self.dataIn is None or not self.dataIn.error:
		66	self.run(**kwargs)
		67	elif self.dataIn.error:
		68	self.dataOut.error = self.dataIn.error
		69	self.dataOut.flagNoData = True
		70	except:
		71	err = traceback.format_exc()
		72	if 'SchainWarning' in err:
		73	log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), self.name)
		74	elif 'SchainError' in err:
		75	log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), self.name)
		76	else:
		77	log.error(err, self.name)
		78	self.dataOut.error = True
99		79
100	**kwargs : Diccionario de argumentos de la funcion a ejecutar	80	for op, optype, opkwargs in self.operations:
101	"""	81	if optype == 'other' and not self.dataOut.flagNoData:
		82	self.dataOut = op.run(self.dataOut, **opkwargs)
		83	elif optype == 'external' and not self.dataOut.flagNoData:
		84	op.queue.put(self.dataOut)
		85	elif optype == 'external' and self.dataOut.error:
		86	op.queue.put(self.dataOut)
102		87
103	raise NotImplementedError	88	return 'Error' if self.dataOut.error else self.dataOut.isReady()
104		89
105	def setup(self):	90	def setup(self):
106		91
107	raise NotImplementedError	92	raise NotImplementedError
108		93
109	def run(self):	94	def run(self):
110		95
111	raise NotImplementedError	96	raise NotImplementedError
112		97
113	def close(self):	98	def close(self):
114		99
115	return	100	return
116		101
117		102
118	class Operation(object):	103	class Operation(object):
119		104
120	"""	105	'''
121	Update - Jan 2018 - MULTIPROCESSING	106	'''
122
123	Most of the methods remained the same. The decorator parse the arguments and executed the run() method for each process.
124	The constructor doe snot receive any argument, neither the baseclass.
125
126
127	Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
128	y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
129	acumulacion dentro de esta clase
130
131	Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
132		107
133	"""
134	proc_type = 'operation'	108	proc_type = 'operation'
135	__attrs__ = []
136		109
137	def __init__(self):	110	def __init__(self):
138		111
139	self.id = None	112	self.id = None
140	self.isConfig = False	113	self.isConfig = False
141		114
142	if not hasattr(self, 'name'):	115	if not hasattr(self, 'name'):
143	self.name = self.__class__.__name__	116	self.name = self.__class__.__name__
144		117
145	def getAllowedArgs(self):	118	def getAllowedArgs(self):
146	if hasattr(self, '__attrs__'):	119	if hasattr(self, '__attrs__'):
147	return self.__attrs__	120	return self.__attrs__
148	else:	121	else:
149	return inspect.getargspec(self.run).args	122	return inspect.getargspec(self.run).args
150		123
151	def setup(self):	124	def setup(self):
152		125
153	self.isConfig = True	126	self.isConfig = True
154		127
155	raise NotImplementedError	128	raise NotImplementedError
156		129
157	def run(self, dataIn, **kwargs):	130	def run(self, dataIn, **kwargs):
158	"""	131	"""
159	Realiza las operaciones necesarias sobre la dataIn.data y actualiza los	132	Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
160	atributos del objeto dataIn.	133	atributos del objeto dataIn.
161		134
162	Input:	135	Input:
163		136
164	dataIn : objeto del tipo JROData	137	dataIn : objeto del tipo JROData
165		138
166	Return:	139	Return:
167		140
168	None	141	None
169		142
170	Affected:	143	Affected:
171	__buffer : buffer de recepcion de datos.	144	__buffer : buffer de recepcion de datos.
172		145
173	"""	146	"""
174	if not self.isConfig:	147	if not self.isConfig:
175	self.setup(**kwargs)	148	self.setup(**kwargs)
176		149
177	raise NotImplementedError	150	raise NotImplementedError
178		151
179	def close(self):	152	def close(self):
180		153
181	return	154	return
182		155
183	class InputQueue(Thread):
184
185	'''
186	Class to hold input data for Proccessing Units and external Operations,
187	'''
188
189	def __init__(self, project_id, inputId, lock=None):
190
191	Thread.__init__(self)
192	self.queue = Queue()
193	self.project_id = project_id
194	self.inputId = inputId
195	self.lock = lock
196	self.islocked = False
197	self.size = 0
198
199	def run(self):
200
201	c = zmq.Context()
202	self.receiver = c.socket(zmq.SUB)
203	self.receiver.connect(
204	'ipc:///tmp/schain/{}_pub'.format(self.project_id))
205	self.receiver.setsockopt(zmq.SUBSCRIBE, self.inputId.encode())
206
207	while True:
208	obj = self.receiver.recv_multipart()[1]
209	self.size += sys.getsizeof(obj)
210	self.queue.put(obj)
211
212	def get(self):
213
214	if not self.islocked and self.size/1000000 > 512:
215	self.lock.n.value += 1
216	self.islocked = True
217	self.lock.clear()
218	elif self.islocked and self.size/1000000 <= 512:
219	self.islocked = False
220	self.lock.n.value -= 1
221	if self.lock.n.value == 0:
222	self.lock.set()
223
224	obj = self.queue.get()
225	self.size -= sys.getsizeof(obj)
226	return pickle.loads(obj)
227
228		156
229	def MPDecorator(BaseClass):	157	def MPDecorator(BaseClass):
230	"""	158	"""
231	Multiprocessing class decorator	159	Multiprocessing class decorator
232		160
233	This function add multiprocessing features to a BaseClass. ~~Also,~~ ~~it handle~~	161	This function add multiprocessing features to a BaseClass.
234	the communication beetween processes (readers, procUnits and operations).
235	"""	162	"""
236		163
237	class MPClass(BaseClass, Process):	164	class MPClass(BaseClass, Process):
238		165
239	def __init__(self, args, *kwargs):	166	def __init__(self, args, *kwargs):
240	super(MPClass, self).__init__()	167	super(MPClass, self).__init__()
241	Process.__init__(self)	168	Process.__init__(self)
242	self.operationKwargs = {}	169
243	self.args = args	170	self.args = args
244	self.kwargs = kwargs	171	self.kwargs = kwargs
245	self.sender = None
246	self.receiver = None
247	self.i = 0
248	self.t = time.time()	172	self.t = time.time()
		173	self.op_type = 'external'
249	self.name = BaseClass.__name__	174	self.name = BaseClass.__name__
250	self.__doc__ = BaseClass.__doc__	175	self.__doc__ = BaseClass.__doc__
251		176
252	if 'plot' in self.name.lower() and not self.name.endswith('_'):	177	if 'plot' in self.name.lower() and not self.name.endswith('_'):
253	self.name = '{}{}'.format(self.CODE.upper(), 'Plot')	178	self.name = '{}{}'.format(self.CODE.upper(), 'Plot')
254		179
255	self.start_time = time.time()	180	self.start_time = time.time()
256	self.id = args[0]
257	self.inputId = args[1]
258	self.project_id = args[2]
259	self.err_queue = args[3]	181	self.err_queue = args[3]
260	self.~~lock~~ = ~~args~~[4]	182	self.queue = Queue(maxsize=1)
261	self.~~typeProc~~ = ~~args~~[5]	183	self.myrun = BaseClass.run
262	self.err_queue.put('#_start_#')
263	if self.inputId is not None:
264	self.queue = InputQueue(self.project_id, self.inputId, self.lock)
265
266	def subscribe(self):
267	'''
268	Start the zmq socket receiver and subcribe to input ID.
269	'''
270
271	self.queue.start()
272		184
273	def ~~liste~~n(self):	185	def run(self):
274	'''
275	This function waits for objects
276	'''
277
278	return self.queue.get()
279
280	def set_publisher(self):
281	'''
282	This function create a zmq socket for publishing objects.
283	'''
284
285	time.sleep(0.5)
286
287	c = zmq.Context()
288	self.sender = c.socket(zmq.PUB)
289	self.sender.connect(
290	'ipc:///tmp/schain/{}_sub'.format(self.project_id))
291
292	def publish(self, data, id):
293	'''
294	This function publish an object, to an specific topic.
295	It blocks publishing when receiver queue is full to avoid data loss
296	'''
297
298	if self.inputId is None:
299	self.lock.wait()
300	self.sender.send_multipart([str(id).encode(), pickle.dumps(data)])
301
302	def runReader(self):
303	'''
304	Run fuction for read units
305	'''
306	while True:
307
308	try:
309	BaseClass.run(self, **self.kwargs)
310	except:
311	err = traceback.format_exc()
312	if 'No more files' in err:
313	log.warning('No more files to read', self.name)
314	else:
315	self.err_queue.put('{}\|{}'.format(self.name, err))
316	self.dataOut.error = True
317
318	for op, optype, opId, kwargs in self.operations:
319	if optype == 'self' and not self.dataOut.flagNoData:
320	op(**kwargs)
321	elif optype == 'other' and not self.dataOut.flagNoData:
322	self.dataOut = op.run(self.dataOut, **self.kwargs)
323	elif optype == 'external':
324	self.publish(self.dataOut, opId)
325
326	if self.dataOut.flagNoData and not self.dataOut.error:
327	continue
328
329	self.publish(self.dataOut, self.id)
330
331	if self.dataOut.error:
332	break
333
334	time.sleep(0.5)
335
336	def runProc(self):
337	'''
338	Run function for proccessing units
339	'''
340
341	while True:
342	self.dataIn = self.listen()
343
344	if self.dataIn.flagNoData and self.dataIn.error is None:
345	continue
346	elif not self.dataIn.error:
347	try:
348	BaseClass.run(self, **self.kwargs)
349	except:
350	self.err_queue.put('{}\|{}'.format(self.name, traceback.format_exc()))
351	self.dataOut.error = True
352	elif self.dataIn.error:
353	self.dataOut.error = self.dataIn.error
354	self.dataOut.flagNoData = True
355
356	for op, optype, opId, kwargs in self.operations:
357	if optype == 'self' and not self.dataOut.flagNoData:
358	op(**kwargs)
359	elif optype == 'other' and not self.dataOut.flagNoData:
360	self.dataOut = op.run(self.dataOut, **kwargs)
361	elif optype == 'external' and not self.dataOut.flagNoData:
362	self.publish(self.dataOut, opId)
363
364	self.publish(self.dataOut, self.id)
365	for op, optype, opId, kwargs in self.operations:
366	if optype == 'external' and self.dataOut.error:
367	self.publish(self.dataOut, opId)
368
369	if self.dataOut.error:
370	break
371
372	time.sleep(0.5)
373
374	def runOp(self):
375	'''
376	Run function for external operations (this operations just receive data
377	ex: plots, writers, publishers)
378	'''
379		186
380	while True:	187	while True:
381		188
382	dataOut = self.~~listen~~()	189	dataOut = self.queue.get()
383		190
384	if not dataOut.error:	191	if not dataOut.error:
385	try:	192	try:
386	BaseClass.run(self, dataOut, **self.kwargs)	193	BaseClass.run(self, dataOut, **self.kwargs)
387	except:	194	except:
388	~~self~~.~~err_queue~~.~~put~~(~~'{}\|{}'~~.~~format~~(~~self~~.~~name~~, traceback.format_exc()))	195	err = traceback.format_exc()
389	dataOut.error = True	196	log.error(err.split('\n')[-2], self.name)
390	else:	197	else:
391	break	198	break
392		199
393	def run(self):
394	if self.typeProc is "ProcUnit":
395
396	if self.inputId is not None:
397	self.subscribe()
398
399	self.set_publisher()
400
401	if 'Reader' not in BaseClass.__name__:
402	self.runProc()
403	else:
404	self.runReader()
405
406	elif self.typeProc is "Operation":
407
408	self.subscribe()
409	self.runOp()
410
411	else:
412	raise ValueError("Unknown type")
413
414	self.close()	200	self.close()
415		201
416	def close(self):	202	def close(self):
417		203
418	BaseClass.close(self)	204	BaseClass.close(self)
419	self.err_queue.put('#_end_#')
420
421	if self.sender:
422	self.sender.close()
423
424	if self.receiver:
425	self.receiver.close()
426
427	log.success('Done...(Time:{:4.2f} secs)'.format(time.time()-self.start_time), self.name)	205	log.success('Done...(Time:{:4.2f} secs)'.format(time.time()-self.start_time), self.name)
428		206
429	return MPClass	207	return MPClass

schainpy/model/proc/jroproc_correlation.py +1 -1

             import numpy
             from .jroproc_base import ProcessingUnit, Operation
-            from schainpy.model.data.jrodata import Correlation, hildebrand_sekhon
+            from schainpy.model.data.jrodata import Correlation
             class CorrelationProc(ProcessingUnit):
                 pairsList = None
                 data_cf = None
                 def __init__(self, **kwargs):
                     ProcessingUnit.__init__(self, **kwargs)
                     self.objectDict = {}
                     self.buffer = None
                     self.firstdatatime = None
                     self.profIndex = 0
                     self.dataOut = Correlation()
                 def __updateObjFromVoltage(self):
                     self.dataOut.timeZone = self.dataIn.timeZone
                     self.dataOut.dstFlag = self.dataIn.dstFlag
                     self.dataOut.errorCount = self.dataIn.errorCount
                     self.dataOut.useLocalTime = self.dataIn.useLocalTime
                     self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
                     self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
                     self.dataOut.channelList = self.dataIn.channelList
                     self.dataOut.heightList = self.dataIn.heightList
                     self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
             #         self.dataOut.nHeights = self.dataIn.nHeights
             #         self.dataOut.nChannels = self.dataIn.nChannels
                     self.dataOut.nBaud = self.dataIn.nBaud
                     self.dataOut.nCode = self.dataIn.nCode
                     self.dataOut.code = self.dataIn.code
             #        self.dataOut.nProfiles = self.dataOut.nFFTPoints
                     self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
                     self.dataOut.utctime = self.firstdatatime
                     self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
                     self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
                     self.dataOut.nCohInt = self.dataIn.nCohInt
             #        self.dataOut.nIncohInt = 1
                     self.dataOut.ippSeconds = self.dataIn.ippSeconds
                     self.dataOut.nProfiles = self.dataIn.nProfiles
                     self.dataOut.utctime = self.dataIn.utctime
             #        self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
             #         self.dataOut.timeInterval = self.dataIn.timeInterval*self.dataOut.nPoints
                 def removeDC(self, jspectra):
                     nChannel = jspectra.shape[0]
                     for i in range(nChannel):
                         jspectra_tmp = jspectra[i,:,:]
                         jspectra_DC = numpy.mean(jspectra_tmp,axis = 0)
                         jspectra_tmp = jspectra_tmp - jspectra_DC
                         jspectra[i,:,:] = jspectra_tmp
                     return jspectra
                 def removeNoise(self, mode = 2):
                     indR = numpy.where(self.dataOut.lagR == 0)[0][0]
                     indT = numpy.where(self.dataOut.lagT == 0)[0][0]
                     jspectra = self.dataOut.data_corr[:,:,indR,:]
                     num_chan = jspectra.shape[0]
                     num_hei = jspectra.shape[2]
                     freq_dc = indT
                     ind_vel = numpy.array([-2,-1,1,2]) + freq_dc
                     NPot = self.dataOut.getNoise(mode)
                     jspectra[:,freq_dc,:] = jspectra[:,freq_dc,:] - NPot
                     SPot = jspectra[:,freq_dc,:]
                     pairsAutoCorr = self.dataOut.getPairsAutoCorr()
             #         self.dataOut.signalPotency = SPot
                     self.dataOut.noise = NPot
                     self.dataOut.SNR = (SPot/NPot)[pairsAutoCorr]
                     self.dataOut.data_corr[:,:,indR,:] = jspectra
                     return 1
                 def run(self, lags=None, mode = 'time',  pairsList=None, fullBuffer=False, nAvg = 1, removeDC = False, splitCF=False):
                     self.dataOut.flagNoData = True
                     if self.dataIn.type == "Correlation":
                         self.dataOut.copy(self.dataIn)
                         return
                     if self.dataIn.type == "Voltage":
                         nChannels = self.dataIn.nChannels
                         nProfiles = self.dataIn.nProfiles
                         nHeights = self.dataIn.nHeights
                         data_pre = self.dataIn.data
                         #---------------    Remover DC    ------------
                         if removeDC:
                             data_pre = self.removeDC(data_pre)
                         #---------------------------------------------
             #             pairsList = list(ccfList)
             #             for i in acfList:
             #                 pairsList.append((i,i))
             #
             #             ccf_pairs = numpy.arange(len(ccfList))
             #             acf_pairs = numpy.arange(len(ccfList),len(pairsList))
                         self.__updateObjFromVoltage()
                         #----------------------------------------------------------------------
                         #Creating temporal buffers
                         if fullBuffer:
                             tmp = numpy.zeros((len(pairsList), len(lags), nProfiles, nHeights), dtype = 'complex')*numpy.nan
                         elif mode == 'time':
                             if lags == None:
                                 lags = numpy.arange(-nProfiles+1, nProfiles)
                             tmp = numpy.zeros((len(pairsList), len(lags), nHeights),dtype='complex')
                         elif mode == 'height':
                             if lags == None:
                                 lags = numpy.arange(-nHeights+1, nHeights)
                             tmp = numpy.zeros(len(pairsList), (len(lags), nProfiles),dtype='complex')
                         #For loop
                         for l in range(len(pairsList)):
                             ch0 = pairsList[l][0]
                             ch1 = pairsList[l][1]
                             for i in range(len(lags)):
                                 idx = lags[i]
                                 if idx >= 0:
                                     if mode == 'time':
                                         ccf0 = data_pre[ch0,:nProfiles-idx,:]*numpy.conj(data_pre[ch1,idx:,:]) #time
                                     else:
                                         ccf0 = data_pre[ch0,:,nHeights-idx]*numpy.conj(data_pre[ch1,:,idx:])   #heights
                                 else:
                                     if mode == 'time':
                                         ccf0 = data_pre[ch0,-idx:,:]*numpy.conj(data_pre[ch1,:nProfiles+idx,:])  #time
                                     else:
                                         ccf0 = data_pre[ch0,:,-idx:]*numpy.conj(data_pre[ch1,:,:nHeights+idx])   #heights
                                 if fullBuffer:
                                     tmp[l,i,:ccf0.shape[0],:] = ccf0
                                 else:
                                     tmp[l,i,:] = numpy.sum(ccf0, axis=0)
                         #-----------------------------------------------------------------
                         if fullBuffer:
                             tmp = numpy.sum(numpy.reshape(tmp,(tmp.shape[0],tmp.shape[1],tmp.shape[2]/nAvg,nAvg,tmp.shape[3])),axis=3)
                             self.dataOut.nAvg = nAvg
                         self.dataOut.data_cf = tmp
                         self.dataOut.mode = mode
                         self.dataOut.nLags = len(lags)
                         self.dataOut.pairsList = pairsList
                         self.dataOut.nPairs = len(pairsList)
                         #Se Calcula los factores de Normalizacion
                         if mode == 'time':
                             delta = self.dataIn.ippSeconds*self.dataIn.nCohInt
                         else:
                             delta = self.dataIn.heightList[1] - self.dataIn.heightList[0]
                         self.dataOut.lagRange = numpy.array(lags)*delta
             #             self.dataOut.nCohInt = self.dataIn.nCohInt*nAvg
                         self.dataOut.flagNoData = False
             #             a = self.dataOut.normFactor
                         return

schainpy/model/proc/jroproc_heispectra.py +1 -1

             import numpy
             from .jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jrodata import SpectraHeis
             from schainpy.utils import log
-            @MPDecorator
             class SpectraHeisProc(ProcessingUnit):
                 def __init__(self):#, **kwargs):
                     ProcessingUnit.__init__(self)#, **kwargs)
             #        self.buffer = None
             #        self.firstdatatime = None
             #        self.profIndex = 0
                     self.dataOut = SpectraHeis()
                 def __updateObjFromVoltage(self):
                     self.dataOut.timeZone = self.dataIn.timeZone
                     self.dataOut.dstFlag = self.dataIn.dstFlag
                     self.dataOut.errorCount = self.dataIn.errorCount
                     self.dataOut.useLocalTime = self.dataIn.useLocalTime
                     self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()#
                     self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()#
                     self.dataOut.channelList = self.dataIn.channelList
                     self.dataOut.heightList = self.dataIn.heightList
             #        self.dataOut.dtype = self.dataIn.dtype
                     self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
             #        self.dataOut.nHeights = self.dataIn.nHeights
             #        self.dataOut.nChannels = self.dataIn.nChannels
                     self.dataOut.nBaud = self.dataIn.nBaud
                     self.dataOut.nCode = self.dataIn.nCode
                     self.dataOut.code = self.dataIn.code
             #        self.dataOut.nProfiles = 1
                     self.dataOut.ippFactor = 1
                     self.dataOut.noise_estimation = None
             #        self.dataOut.nProfiles = self.dataOut.nFFTPoints
                     self.dataOut.nFFTPoints = self.dataIn.nHeights
             #        self.dataOut.channelIndexList = self.dataIn.channelIndexList
             #        self.dataOut.flagNoData = self.dataIn.flagNoData
                     self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
                     self.dataOut.utctime = self.dataIn.utctime
             #        self.dataOut.utctime = self.firstdatatime
                     self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
                     self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
             #        self.dataOut.flagShiftFFT = self.dataIn.flagShiftFFT
                     self.dataOut.nCohInt = self.dataIn.nCohInt
                     self.dataOut.nIncohInt = 1
             #         self.dataOut.ippSeconds= self.dataIn.ippSeconds
                     self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
             #         self.dataOut.timeInterval = self.dataIn.timeInterval*self.dataOut.nIncohInt
             #        self.dataOut.set=self.dataIn.set
             #        self.dataOut.deltaHeight=self.dataIn.deltaHeight
                 def __updateObjFromFits(self):
                     self.dataOut.utctime = self.dataIn.utctime
             #         self.dataOut.channelIndexList = self.dataIn.channelIndexList
                     self.dataOut.channelList = self.dataIn.channelList
                     self.dataOut.heightList = self.dataIn.heightList
                     self.dataOut.data_spc = self.dataIn.data
                     self.dataOut.ippSeconds = self.dataIn.ippSeconds
                     self.dataOut.nCohInt = self.dataIn.nCohInt
                     self.dataOut.nIncohInt = self.dataIn.nIncohInt
             #         self.dataOut.timeInterval = self.dataIn.timeInterval
                     self.dataOut.timeZone = self.dataIn.timeZone
                     self.dataOut.useLocalTime = True
             #         self.dataOut.
             #         self.dataOut.
                 def __getFft(self):
                     fft_volt = numpy.fft.fft(self.dataIn.data, axis=1)
                     fft_volt = numpy.fft.fftshift(fft_volt,axes=(1,))
                     spc = numpy.abs(fft_volt * numpy.conjugate(fft_volt))/(self.dataOut.nFFTPoints)
                     self.dataOut.data_spc = spc
                 def run(self):
                     self.dataOut.flagNoData = True
                     if self.dataIn.type == "Fits":
                         self.__updateObjFromFits()
                         self.dataOut.flagNoData = False
                         return
                     if self.dataIn.type == "SpectraHeis":
                         self.dataOut.copy(self.dataIn)
                         return
                     if self.dataIn.type == "Voltage":
                         self.__updateObjFromVoltage()
                         self.__getFft()
                         self.dataOut.flagNoData = False
                         return
                     raise ValueError("The type object %s is not valid"%(self.dataIn.type))
                 def selectChannels(self, channelList):
                     channelIndexList = []
                     for channel in channelList:
                         index = self.dataOut.channelList.index(channel)
                         channelIndexList.append(index)
                     self.selectChannelsByIndex(channelIndexList)
                 def selectChannelsByIndex(self, channelIndexList):
                     """
                     Selecciona un bloque de datos en base a canales segun el channelIndexList
                     Input:
                         channelIndexList    :    lista sencilla de canales a seleccionar por ej. [2,3,7]
                     Affected:
                         self.dataOut.data
                         self.dataOut.channelIndexList
                         self.dataOut.nChannels
                         self.dataOut.m_ProcessingHeader.totalSpectra
                         self.dataOut.systemHeaderObj.numChannels
                         self.dataOut.m_ProcessingHeader.blockSize
                     Return:
                         None
                     """
                     for channelIndex in channelIndexList:
                         if channelIndex not in self.dataOut.channelIndexList:
                             print(channelIndexList)
                             raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
             #         nChannels = len(channelIndexList)
                     data_spc = self.dataOut.data_spc[channelIndexList,:]
                     self.dataOut.data_spc = data_spc
                     self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
                     return 1
             class IncohInt4SpectraHeis(Operation):
                 isConfig = False
                 __profIndex = 0
                 __withOverapping  = False
                 __byTime = False
                 __initime = None
                 __lastdatatime = None
                 __integrationtime = None
                 __buffer = None
                 __dataReady = False
                 n = None
                 def __init__(self):#, **kwargs):
                     Operation.__init__(self)#, **kwargs)
             #         self.isConfig = False
                 def setup(self, n=None, timeInterval=None, overlapping=False):
                     """
                     Set the parameters of the integration class.
                     Inputs:
                         n        :    Number of coherent integrations
                         timeInterval   :    Time of integration. If the parameter "n" is selected this one does not work
                         overlapping    :
                     """
                     self.__initime = None
                     self.__lastdatatime = 0
                     self.__buffer = None
                     self.__dataReady = False
                     if n == None and timeInterval == None:
                         raise ValueError("n or timeInterval should be specified ...")
                     if n != None:
                         self.n = n
                         self.__byTime = False
                     else:
                         self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
                         self.n = 9999
                         self.__byTime = True
                     if overlapping:
                         self.__withOverapping = True
                         self.__buffer = None
                     else:
                         self.__withOverapping = False
                         self.__buffer = 0
                     self.__profIndex = 0
                 def putData(self, data):
                     """
                     Add a profile to the __buffer and increase in one the __profileIndex
                     """
                     if not self.__withOverapping:
                         self.__buffer += data.copy()
                         self.__profIndex += 1
                         return
                     #Overlapping data
                     nChannels, nHeis = data.shape
                     data = numpy.reshape(data, (1, nChannels, nHeis))
                     #If the buffer is empty then it takes the data value
                     if self.__buffer is None:
                         self.__buffer = data
                         self.__profIndex += 1
                         return
                     #If the buffer length is lower than n then stakcing the data value
                     if self.__profIndex < self.n:
                         self.__buffer = numpy.vstack((self.__buffer, data))
                         self.__profIndex += 1
                         return
                     #If the buffer length is equal to n then replacing the last buffer value with the data value
                     self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
                     self.__buffer[self.n-1] = data
                     self.__profIndex = self.n
                     return
                 def pushData(self):
                     """
                     Return the sum of the last profiles and the profiles used in the sum.
                     Affected:
                     self.__profileIndex
                     """
                     if not self.__withOverapping:
                         data = self.__buffer
                         n = self.__profIndex
                         self.__buffer = 0
                         self.__profIndex = 0
                         return data, n
                     #Integration with Overlapping
                     data = numpy.sum(self.__buffer, axis=0)
                     n = self.__profIndex
                     return data, n
                 def byProfiles(self, data):
                     self.__dataReady = False
                     avgdata = None
             #         n = None
                     self.putData(data)
                     if self.__profIndex == self.n:
                         avgdata, n = self.pushData()
                         self.__dataReady = True
                     return avgdata
                 def byTime(self, data, datatime):
                     self.__dataReady = False
                     avgdata = None
                     n = None
                     self.putData(data)
                     if (datatime - self.__initime) >= self.__integrationtime:
                         avgdata, n = self.pushData()
                         self.n = n
                         self.__dataReady = True
                     return avgdata
                 def integrate(self, data, datatime=None):
                     if self.__initime == None:
                         self.__initime = datatime
                     if self.__byTime:
                         avgdata = self.byTime(data, datatime)
                     else:
                         avgdata = self.byProfiles(data)
                     self.__lastdatatime = datatime
                     if avgdata is None:
                         return None, None
                     avgdatatime = self.__initime
                     deltatime = datatime -self.__lastdatatime
                     if not self.__withOverapping:
                         self.__initime = datatime
                     else:
                         self.__initime += deltatime
                     return avgdata, avgdatatime
                 def run(self, dataOut, n=None, timeInterval=None, overlapping=False, **kwargs):
                     if not self.isConfig:
                         self.setup(n=n, timeInterval=timeInterval, overlapping=overlapping)
                         self.isConfig = True
                     avgdata, avgdatatime = self.integrate(dataOut.data_spc, dataOut.utctime)
             #        dataOut.timeInterval *= n
                     dataOut.flagNoData = True
                     if self.__dataReady:
                         dataOut.data_spc = avgdata
                         dataOut.nIncohInt *= self.n
             #            dataOut.nCohInt *= self.n
                         dataOut.utctime = avgdatatime
             #             dataOut.timeInterval = dataOut.ippSeconds * dataOut.nIncohInt
             #            dataOut.timeInterval = self.__timeInterval*self.n
                         dataOut.flagNoData = False
                     return dataOut

schainpy/model/proc/jroproc_parameters.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/jroproc_spectra.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/jroproc_voltage.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages