schain Commit - r1539:03a9c71b14ee · Jicamarca Repository

Merge branch 'v3.0-WR' of http://intranet.igp.gob.pe:8082/schain into v3.0-WR

Juan C. Espinoza -

r1539:03a9c71b14ee

parent child

Context file:

r1539:03a9c71b14ee

Collapse all files

schainpy/scripts/sophy_proc_JC.py created 644 +418 0

@@ -0,0 +1,418
	1	# SOPHY PROC script
	2	import os, sys, json, argparse
	3	import datetime
	4	import time
	5
	6	PATH = '/DATA_RM/DATA'
	7	PATH = '/media/jespinoza/Elements'
	8	PATH = '/media/jespinoza/data/SOPHY'
	9	PATH = '/home/soporte/Documents/EVENTO/'
	10
	11	# NOTA: EL GRABADO ESTA EN PARAM
	12	PARAM = {
	13	'S': {'zmin': -45, 'zmax': -25, 'colormap': 'jet', 'label': 'Power', 'wrname': 'power','cb_label': 'dBm', 'ch':0},
	14	'SNR': {'zmin': -40, 'zmax': -20, 'colormap': 'jet', 'label': 'SNR', 'wrname': 'snr','cb_label': 'dB', 'ch':0},
	15	'V': {'zmin': -12, 'zmax': 12, 'colormap': 'sophy_v', 'label': 'Velocity', 'wrname': 'velocity', 'cb_label': 'm/s', 'ch':0},
	16	'R': {'zmin': 0, 'zmax': 1, 'colormap': 'jet', 'label': 'RhoHV', 'wrname':'rhoHV', 'cb_label': '', 'ch':0},
	17	'P': {'zmin': -180,'zmax': 180,'colormap': 'RdBu_r', 'label': 'PhiDP', 'wrname':'phiDP' , 'cb_label': 'degrees', 'ch':0},
	18	'D': {'zmin': -30, 'zmax': 80, 'colormap': 'sophy_r','label': 'ZDR','wrname':'differential_reflectivity' , 'cb_label': 'dBz','ch':0},
	19	'Z': {'zmin': -30, 'zmax': 80, 'colormap': 'sophy_r','label': 'Reflectivity ', 'wrname':'reflectivity', 'cb_label': 'dBz','ch':0},
	20	'W': {'zmin': 0, 'zmax': 15, 'colormap': 'sophy_w','label': 'Spectral Width', 'wrname':'spectral_width', 'cb_label': 'm/s', 'ch':0}
	21	}
	22
	23	META = ['heightList', 'data_azi', 'data_ele', 'mode_op', 'latitude', 'longitude', 'altitude', 'heading', 'radar_name',
	24	'institution', 'contact', 'h0', 'range_unit', 'prf', 'prf_unit', 'variable', 'variable_unit', 'n_pulses',
	25	'pulse1_range', 'pulse1_width', 'pulse2_width', 'pulse1_repetitions', 'pulse2_repetitions', 'pulse_width_unit',
	26	'snr_threshold','dataPP_NOISE']
	27
	28
	29	def max_index(r, sample_rate, ipp):
	30
	31	return int(sample_rateipp1e6 * r / 60) + int(sample_rateipp1e6 * 1.2 / 60)
	32
	33	def main(args):
	34
	35	experiment = args.experiment
	36	fp = open(os.path.join(PATH, experiment, 'experiment.conf'))
	37	conf = json.loads(fp.read())
	38
	39	ipp_km = conf['usrp_tx']['ipp']
	40	ipp = ipp_km * 2 /300000
	41	sample_rate = conf['usrp_rx']['sample_rate']
	42	axis = ['0' if x=='elevation' else '1' for x in conf['pedestal']['axis']] # AZIMUTH 1 ELEVACION 0
	43	speed_axis = conf['pedestal']['speed']
	44	steps = conf['pedestal']['table']
	45	time_offset = args.time_offset
	46	parameters = args.parameters
	47	start_date = experiment.split('@')[1].split('T')[0].replace('-', '/')
	48	end_date = start_date
	49	if args.start_time:
	50	start_time = args.start_time
	51	else:
	52	start_time = experiment.split('@')[1].split('T')[1].replace('-', ':')
	53	end_time = '23:59:59'
	54	N = int(1/(speed_axis[0]*ipp)) # 1 GRADO DE RESOLUCION
	55	path = os.path.join(PATH, experiment, 'rawdata')
	56	path_ped = os.path.join(PATH, experiment, 'position')
	57	if args.label:
	58	label = '-{}'.format(args.label)
	59	else:
	60	label = ''
	61	path_plots = os.path.join(PATH, experiment, 'plots{}'.format(label))
	62	path_save = os.path.join(PATH, experiment, 'param{}'.format(label))
	63	RMIX = 1.62
	64	H0 = -1.68
	65	MASK = 0.3
	66
	67	from schainpy.controller import Project
	68
	69	project = Project()
	70	project.setup(id='1', name='Sophy', description='sophy proc')
	71
	72	reader = project.addReadUnit(datatype='DigitalRFReader',
	73	path=path,
	74	startDate=start_date,
	75	endDate=end_date,
	76	startTime=start_time,
	77	endTime=end_time,
	78	delay=30,
	79	online=args.online,
	80	walk=1,
	81	ippKm = ipp_km,
	82	getByBlock = 1,
	83	nProfileBlocks = N,
	84	)
	85
	86	if not conf['usrp_tx']['enable_2']: # One Pulse
	87	n_pulses = 1
	88	pulse_1_width = conf['usrp_tx']['pulse_1']
	89	pulse_1_repetitions = conf['usrp_tx']['repetitions_1']
	90	pulse_2_width = 0
	91	pulse_2_repetitions = 0
	92
	93	voltage = project.addProcUnit(datatype='VoltageProc', inputId=reader.getId())
	94
	95	if conf['usrp_tx']['code_type_1'] != 'None':
	96	codes = [ c.strip() for c in conf['usrp_tx']['code_1'].split(',')]
	97	code = []
	98	for c in codes:
	99	code.append([int(x) for x in c])
	100	op = voltage.addOperation(name='Decoder', optype='other')
	101	op.addParameter(name='code', value=code)
	102	op.addParameter(name='nCode', value=len(code), format='int')
	103	op.addParameter(name='nBaud', value=len(code[0]), format='int')
	104
	105	op = voltage.addOperation(name='CohInt', optype='other') #Minimo integrar 2 perfiles por ser codigo complementario
	106	op.addParameter(name='n', value=len(code), format='int')
	107	ncode = len(code)
	108	else:
	109	ncode = 1
	110	code = ['0']
	111
	112	op = voltage.addOperation(name='setH0')
	113	op.addParameter(name='h0', value=H0)
	114
	115	if args.range > 0:
	116	op = voltage.addOperation(name='selectHeights')
	117	op.addParameter(name='minIndex', value='0', format='int')
	118	op.addParameter(name='maxIndex', value=max_index(args.range, sample_rate, ipp), format='int')
	119
	120	op = voltage.addOperation(name='PulsePair_vRF', optype='other')
	121	op.addParameter(name='n', value=int(N)/ncode, format='int')
	122
	123	proc = project.addProcUnit(datatype='ParametersProc', inputId=voltage.getId())
	124
	125	opObj10 = proc.addOperation(name="WeatherRadar")
	126	opObj10.addParameter(name='tauW',value=(1e-6/sample_rate)*len(code[0]))
	127	opObj10.addParameter(name='Pt',value=((1e-6/sample_rate)len(code[0])/ipp)200)
	128
	129	op = proc.addOperation(name='PedestalInformation')
	130	op.addParameter(name='path', value=path_ped, format='str')
	131	op.addParameter(name='interval', value='0.04')
	132	op.addParameter(name='time_offset', value=time_offset)
	133	op.addParameter(name='mode', value='PPI')
	134
	135	for param in parameters:
	136	op = proc.addOperation(name='Block360')
	137	op.addParameter(name='runNextOp', value=True)
	138
	139	op= proc.addOperation(name='WeatherParamsPlot')
	140	if args.save: op.addParameter(name='save', value=path_plots, format='str')
	141	op.addParameter(name='save_period', value=-1)
	142	op.addParameter(name='show', value=args.show)
	143	op.addParameter(name='channels', value='1,')
	144	op.addParameter(name='zmin', value=PARAM[param]['zmin'])
	145	op.addParameter(name='zmax', value=PARAM[param]['zmax'])
	146	op.addParameter(name='attr_data', value=param, format='str')
	147	op.addParameter(name='labels', value=[PARAM[param]['label']])
	148	op.addParameter(name='save_code', value=param)
	149	op.addParameter(name='cb_label', value=PARAM[param]['cb_label'])
	150	op.addParameter(name='colormap', value=PARAM[param]['colormap'])
	151	op.addParameter(name='bgcolor', value='black')
	152	if MASK: op.addParameter(name='mask', value=MASK, format='float')
	153	if args.server:
	154	op.addParameter(name='server', value='0.0.0.0:4444')
	155	op.addParameter(name='exp_code', value='400')
	156
	157	desc = {
	158	'Data': {
	159	param: PARAM[param]['wrname'],
	160	'utctime': 'time'
	161	},
	162	'Metadata': {
	163	'heightList': 'range',
	164	'data_azi': 'azimuth',
	165	'data_ele': 'elevation',
	166	'mode_op': 'scan_type',
	167	'h0': 'range_correction',
	168	}
	169	}
	170
	171	if args.save:
	172	writer = merge.addOperation(name='HDFWriter')
	173	writer.addParameter(name='path', value=path_save, format='str')
	174	writer.addParameter(name='Reset', value=True)
	175	writer.addParameter(name='setType', value='weather')
	176	writer.addParameter(name='description', value=json.dumps(desc))
	177	writer.addParameter(name='blocksPerFile', value='1',format='int')
	178	writer.addParameter(name='metadataList', value=','.join(META))
	179	writer.addParameter(name='dataList', value='data_param,utctime')
	180	writer.addParameter(name='weather_var', value=param)
	181	writer.addParameter(name='mask', value=MASK, format='float')
	182	# meta
	183	writer.addParameter(name='latitude', value='-12.040436')
	184	writer.addParameter(name='longitude', value='-75.295893')
	185	writer.addParameter(name='altitude', value='3379.2147')
	186	writer.addParameter(name='heading', value='0')
	187	writer.addParameter(name='radar_name', value='SOPHy')
	188	writer.addParameter(name='institution', value='IGP')
	189	writer.addParameter(name='contact', value='dscipion@igp.gob.pe')
	190	writer.addParameter(name='created_by', value='Signal Chain (https://pypi.org/project/schainpy/)')
	191	writer.addParameter(name='range_unit', value='km')
	192	writer.addParameter(name='prf', value=1/ipp)
	193	writer.addParameter(name='prf_unit', value='hertz')
	194	writer.addParameter(name='variable', value=PARAM[param]['label'])
	195	writer.addParameter(name='variable_unit', value=PARAM[param]['cb_label'])
	196	writer.addParameter(name='n_pulses', value=n_pulses)
	197	writer.addParameter(name='pulse1_range', value=RMIX)
	198	writer.addParameter(name='pulse1_width', value=pulse_1_width)
	199	writer.addParameter(name='pulse2_width', value=pulse_2_width)
	200	writer.addParameter(name='pulse1_repetitions', value=pulse_1_repetitions)
	201	writer.addParameter(name='pulse2_repetitions', value=pulse_2_repetitions)
	202	writer.addParameter(name='pulse_width_unit', value='microseconds')
	203	writer.addParameter(name='snr_threshold', value=MASK)
	204
	205
	206	else: #Two pulses
	207	n_pulses = 1
	208	pulse_1_width = conf['usrp_tx']['pulse_1']
	209	pulse_1_repetitions = conf['usrp_tx']['repetitions_1']
	210	pulse_2_width = conf['usrp_tx']['pulse_2']
	211	pulse_2_repetitions = conf['usrp_tx']['repetitions_2']
	212
	213	voltage1 = project.addProcUnit(datatype='VoltageProc', inputId=reader.getId())
	214
	215	op = voltage1.addOperation(name='ProfileSelector')
	216	op.addParameter(name='profileRangeList', value='0,{}'.format(conf['usrp_tx']['repetitions_1']-1))
	217
	218	if conf['usrp_tx']['code_type_1'] != 'None':
	219	codes = [ c.strip() for c in conf['usrp_tx']['code_1'].split(',')]
	220	code = []
	221	for c in codes:
	222	code.append([int(x) for x in c])
	223	op = voltage1.addOperation(name='Decoder', optype='other')
	224	op.addParameter(name='code', value=code)
	225	op.addParameter(name='nCode', value=len(code), format='int')
	226	op.addParameter(name='nBaud', value=len(code[0]), format='int')
	227	else:
	228	code = ['0']
	229
	230	op = voltage1.addOperation(name='CohInt', optype='other') #Minimo integrar 2 perfiles por ser codigo complementario
	231	op.addParameter(name='n', value=2, format='int')
	232
	233	if args.range > 0:
	234	op = voltage1.addOperation(name='selectHeights')
	235	op.addParameter(name='minIndex', value='0', format='int')
	236	op.addParameter(name='maxIndex', value=max_index(RMIX, sample_rate, ipp), format='int')
	237
	238	op = voltage1.addOperation(name='setH0')
	239	op.addParameter(name='h0', value=H0, format='float')
	240
	241	op = voltage1.addOperation(name='PulsePair_vRF', optype='other')
	242	op.addParameter(name='n', value=int(conf['usrp_tx']['repetitions_1'])/2, format='int')
	243
	244	proc1 = project.addProcUnit(datatype='ParametersProc', inputId=voltage1.getId())
	245	proc1.addParameter(name='runNextUnit', value=True)
	246
	247	opObj10 = proc1.addOperation(name="WeatherRadar")
	248	opObj10.addParameter(name='tauW',value=(1e-6/sample_rate)*len(code[0]))
	249	opObj10.addParameter(name='Pt',value=((1e-6/sample_rate)len(code[0])/ipp)200)
	250
	251	op = proc1.addOperation(name='PedestalInformation')
	252	op.addParameter(name='path', value=path_ped, format='str')
	253	op.addParameter(name='interval', value='0.04')
	254	op.addParameter(name='time_offset', value=time_offset)
	255	op.addParameter(name='mode', value='PPI')
	256
	257	op = proc1.addOperation(name='Block360')
	258	op.addParameter(name='attr_data', value='data_param')
	259	op.addParameter(name='runNextOp', value=True)
	260
	261
	262	voltage2 = project.addProcUnit(datatype='VoltageProc', inputId=reader.getId())
	263
	264	op = voltage2.addOperation(name='ProfileSelector')
	265	op.addParameter(name='profileRangeList', value='{},{}'.format(conf['usrp_tx']['repetitions_1'], conf['usrp_tx']['repetitions_1']+conf['usrp_tx']['repetitions_2']-1))
	266
	267	if conf['usrp_tx']['code_type_2']:
	268	codes = [ c.strip() for c in conf['usrp_tx']['code_2'].split(',')]
	269	code = []
	270	for c in codes:
	271	code.append([int(x) for x in c])
	272	op = voltage2.addOperation(name='Decoder', optype='other')
	273	op.addParameter(name='code', value=code)
	274	op.addParameter(name='nCode', value=len(code), format='int')
	275	op.addParameter(name='nBaud', value=len(code[0]), format='int')
	276
	277	op = voltage2.addOperation(name='CohInt', optype='other') #Minimo integrar 2 perfiles por ser codigo complementario
	278	op.addParameter(name='n', value=len(code), format='int')
	279	ncode = len(code)
	280	else:
	281	ncode = 1
	282
	283	if args.range > 0:
	284	op = voltage2.addOperation(name='selectHeights')
	285	op.addParameter(name='minIndex', value=max_index(RMIX, sample_rate, ipp), format='int')
	286	op.addParameter(name='maxIndex', value=max_index(args.range, sample_rate, ipp), format='int')
	287
	288	op = voltage2.addOperation(name='setH0')
	289	op.addParameter(name='h0', value=H0, format='float')
	290
	291	op = voltage2.addOperation(name='PulsePair_vRF', optype='other')
	292	op.addParameter(name='n', value=int(conf['usrp_tx']['repetitions_2'])/ncode, format='int')
	293
	294	proc2 = project.addProcUnit(datatype='ParametersProc', inputId=voltage2.getId())
	295	proc2.addParameter(name='runNextUnit', value=True)
	296
	297	opObj10 = proc2.addOperation(name="WeatherRadar")
	298	opObj10.addParameter(name='tauW',value=(1e-6/sample_rate)*len(code[0]))
	299	opObj10.addParameter(name='Pt',value=((1e-6/sample_rate)len(code[0])/ipp)200)
	300
	301	op = proc2.addOperation(name='PedestalInformation')
	302	op.addParameter(name='path', value=path_ped, format='str')
	303	op.addParameter(name='interval', value='0.04')
	304	op.addParameter(name='time_offset', value=time_offset)
	305	op.addParameter(name='mode', value='PPI')
	306
	307	op = proc2.addOperation(name='Block360')
	308	op.addParameter(name='attr_data', value='data_param')
	309	op.addParameter(name='runNextOp', value=True)
	310
	311	merge = project.addProcUnit(datatype='MergeProc', inputId=[proc1.getId(), proc2.getId()])
	312	merge.addParameter(name='attr_data', value='data_param')
	313	merge.addParameter(name='mode', value='7') #RM
	314
	315
	316	for param in parameters:
	317
	318	if args.plot:
	319	op= merge.addOperation(name='WeatherParamsPlot')
	320	if args.save:
	321	op.addParameter(name='save', value=path_plots, format='str')
	322	op.addParameter(name='save_period', value=-1)
	323	op.addParameter(name='show', value=args.show)
	324	op.addParameter(name='channels', value='0,')
	325	op.addParameter(name='zmin', value=PARAM[param]['zmin'], format='int')
	326	op.addParameter(name='zmax', value=PARAM[param]['zmax'], format='int')
	327	op.addParameter(name='attr_data', value=param, format='str')
	328	op.addParameter(name='labels', value=[PARAM[param]['label']])
	329	op.addParameter(name='save_code', value=param)
	330	op.addParameter(name='cb_label', value=PARAM[param]['cb_label'])
	331	op.addParameter(name='colormap', value=PARAM[param]['colormap'])
	332	op.addParameter(name='bgcolor', value='black')
	333	if MASK: op.addParameter(name='mask', value=MASK, format='float')
	334	if args.server:
	335	op.addParameter(name='server', value='0.0.0.0:4444')
	336	op.addParameter(name='exp_code', value='400')
	337
	338	desc = {
	339	'Data': {
	340	'data_param': {PARAM[param]['wrname']: ['H', 'V']},
	341	'utctime': 'time'
	342	},
	343	'Metadata': {
	344	'heightList': 'range',
	345	'data_azi': 'azimuth',
	346	'data_ele': 'elevation',
	347	'mode_op': 'scan_type',
	348	'h0': 'range_correction',
	349	'dataPP_NOISE': 'noise',
	350	}
	351	}
	352
	353	if args.save:
	354	writer = merge.addOperation(name='HDFWriter')
	355	writer.addParameter(name='path', value=path_save, format='str')
	356	writer.addParameter(name='Reset', value=True)
	357	writer.addParameter(name='setType', value='weather')
	358	writer.addParameter(name='description', value=json.dumps(desc))
	359	writer.addParameter(name='blocksPerFile', value='1',format='int')
	360	writer.addParameter(name='metadataList', value=','.join(META))
	361	writer.addParameter(name='dataList', value='data_param,utctime')
	362	writer.addParameter(name='weather_var', value=param)
	363	writer.addParameter(name='mask', value=MASK, format='float')
	364	# meta
	365	writer.addParameter(name='latitude', value='-12.040436')
	366	writer.addParameter(name='longitude', value='-75.295893')
	367	writer.addParameter(name='altitude', value='3379.2147')
	368	writer.addParameter(name='heading', value='0')
	369	writer.addParameter(name='radar_name', value='SOPHy')
	370	writer.addParameter(name='institution', value='IGP')
	371	writer.addParameter(name='contact', value='dscipion@igp.gob.pe')
	372	writer.addParameter(name='created_by', value='Signal Chain (https://pypi.org/project/schainpy/)')
	373	writer.addParameter(name='range_unit', value='km')
	374	writer.addParameter(name='prf', value=1/ipp)
	375	writer.addParameter(name='prf_unit', value='hertz')
	376	writer.addParameter(name='variable', value=PARAM[param]['label'])
	377	writer.addParameter(name='variable_unit', value=PARAM[param]['cb_label'])
	378	writer.addParameter(name='n_pulses', value=n_pulses)
	379	writer.addParameter(name='pulse1_range', value=RMIX)
	380	writer.addParameter(name='pulse1_width', value=pulse_1_width)
	381	writer.addParameter(name='pulse2_width', value=pulse_2_width)
	382	writer.addParameter(name='pulse1_repetitions', value=pulse_1_repetitions)
	383	writer.addParameter(name='pulse2_repetitions', value=pulse_2_repetitions)
	384	writer.addParameter(name='pulse_width_unit', value='microseconds')
	385	writer.addParameter(name='snr_threshold', value=MASK)
	386
	387
	388	project.start()
	389
	390	if __name__ == '__main__':
	391
	392	parser = argparse.ArgumentParser(description='Script to process SOPHy data.')
	393	parser.add_argument('experiment',
	394	help='Experiment name')
	395	parser.add_argument('--parameters', nargs='*', default=['S'],
	396	help='Variables to process: P, Z, V')
	397	parser.add_argument('--time_offset', default=0,
	398	help='Fix time offset')
	399	parser.add_argument('--range', default=0, type=float,
	400	help='Max range to plot')
	401	parser.add_argument('--save', action='store_true',
	402	help='Create output files')
	403	parser.add_argument('--plot', action='store_true',
	404	help='Create plot files')
	405	parser.add_argument('--show', action='store_true',
	406	help='Show matplotlib plot.')
	407	parser.add_argument('--online', action='store_true',
	408	help='Set online mode.')
	409	parser.add_argument('--server', action='store_true',
	410	help='Send to realtime')
	411	parser.add_argument('--start_time', default='',
	412	help='Set start time.')
	413	parser.add_argument('--label', default='',
	414	help='Label for plot & param folder')
	415
	416	args = parser.parse_args()
	417
	418	main(args)

schainpy/scripts/sophy_proc_rev007.py created 644 +117 0

@@ -0,0 +1,117
	1	import os,sys,json
	2	import datetime
	3	import time
	4	import argparse
	5
	6	from schainpy.controller import Project
	7	'''
	8	NOTA:
	9	Este script de prueba.
	10	- Unidad del lectura 'HDFReader'.
	11	- Unidad de procesamiento ParametersProc
	12	'''
	13	PATH = "/home/soporte/Documents/EVENTO/HYO_PM@2022-06-09T15-05-12/paramC0N36.0/2022-06-09T18-00-00/"
	14	#PATH = "/home/soporte/Documents/EVENTO/HYO_PM@2022-06-09T15-05-12/param/2022-06-09T18-00-00/"
	15
	16	#PATH = "/home/soporte/Documents/EVENTO/HYO_PM@2022-06-09T15-05-12/paramC0N36.0/2022-06-09T19-00-00/"
	17	#PATH = "/home/soporte/Documents/EVENTO/HYO_PM@2022-05-31T12-00-17/paramC0N36.0/2022-05-31T16-00-00/"
	18	path = PATH
	19	PARAM = {
	20	'S': {'zmin': -45, 'zmax': -25, 'colormap': 'jet', 'label': 'Power', 'wrname': 'power','cb_label': 'dBm', 'ch':0},
	21	'SNR': {'zmin': -40, 'zmax': -20, 'colormap': 'jet', 'label': 'SNR', 'wrname': 'snr','cb_label': 'dB', 'ch':0},
	22	'V': {'zmin': -12, 'zmax': 12, 'colormap': 'sophy_v', 'label': 'Velocity', 'wrname': 'velocity', 'cb_label': 'm/s', 'ch':0},
	23	'R': {'zmin': 0, 'zmax': 1, 'colormap': 'jet', 'label': 'RhoHV', 'wrname':'rhoHV', 'cb_label': '*', 'ch':0},
	24	'P': {'zmin': -180,'zmax': 180,'colormap': 'RdBu_r', 'label': 'PhiDP', 'wrname':'phiDP' , 'cb_label': 'º', 'ch':0},
	25	'D': {'zmin': -30, 'zmax': 80, 'colormap': 'sophy_r','label': 'ZDR','wrname':'differential_reflectivity' , 'cb_label': 'dBz','ch':0},
	26	'Z': {'zmin': -30, 'zmax': 80, 'colormap': 'sophy_r','label': 'Reflectivity ', 'wrname':'reflectivity', 'cb_label': 'dBz','ch':0},
	27	'W': {'zmin': 0, 'zmax': 15, 'colormap': 'sophy_w','label': 'Spectral Width', 'wrname':'spectral_width', 'cb_label': 'm/s', 'ch':0}
	28	}
	29
	30	def main(args):
	31	#filefmt="*****%Y%m%d%H%M%S*******"
	32	#filefmt="SOPHY_20220609_184620_E8.0_Z"
	33	parameters = args.parameters
	34	grado = args.grado
	35	MASK = None
	36
	37	for param in parameters:
	38	filefmt ="*****%Y%m%d%H%M%S*******"
	39	filter= "_E"+str(grado)+".0_"+param
	40	variable = 'Data/'+PARAM[param]['wrname']+'/H'
	41	desc = {
	42	'Data': {
	43	'data_param': [variable],
	44	'utctime' : 'Data/time'
	45	},
	46	'Metadata': {
	47	'heightList': 'Metadata/range',
	48	'data_azi' : 'Metadata/azimuth',
	49	'data_ele' : 'Metadata/elevation',
	50	'mode_op' : 'Metadata/scan_type',
	51	'h0' : 'Metadata/range_correction',
	52	}
	53	}
	54
	55	project = Project()
	56
	57	project.setup(id='10',name='Test Simulator',description=desc)
	58
	59	readUnitConfObj = project.addReadUnit(datatype='HDFReader',
	60	path=path,
	61	startDate="2022/01/01", #"2020/01/01",#today,
	62	endDate= "2022/12/01", #"2020/12/30",#today,
	63	startTime='00:00:00',
	64	endTime='23:59:59',
	65	delay=0,
	66	#set=0,
	67	online=0,
	68	walk=0,
	69	filefmt=filefmt,
	70	filter=filter,
	71	dparam= 1,
	72	description= json.dumps(desc))#1
	73
	74	proc1 = project.addProcUnit(datatype='ParametersProc',inputId=readUnitConfObj.getId())
	75
	76	if args.plot:
	77	print("plotea")
	78	op= proc1.addOperation(name='WeatherParamsPlot')
	79	if args.save:
	80	op.addParameter(name='save', value=path_plots, format='str')
	81	op.addParameter(name='save_period', value=-1)
	82	op.addParameter(name='show', value=args.show)
	83	op.addParameter(name='channels', value='0,')
	84	op.addParameter(name='zmin', value=PARAM[param]['zmin'], format='int')
	85	op.addParameter(name='zmax', value=PARAM[param]['zmax'], format='int')
	86	op.addParameter(name='attr_data', value=param, format='str')
	87	op.addParameter(name='labels', value=[PARAM[param]['label']])
	88	op.addParameter(name='save_code', value=param)
	89	op.addParameter(name='cb_label', value=PARAM[param]['cb_label'])
	90	op.addParameter(name='colormap', value=PARAM[param]['colormap'])
	91	op.addParameter(name='bgcolor', value='black')
	92	if MASK: op.addParameter(name='mask', value=MASK, format='float')
	93	if args.server:
	94	op.addParameter(name='server', value='0.0.0.0:4444')
	95	op.addParameter(name='exp_code', value='400')
	96	project.start()
	97
	98	if __name__ == '__main__':
	99
	100	parser = argparse.ArgumentParser(description='Script to process SOPHy data.')
	101	parser.add_argument('--parameters', nargs='*', default=['S'],
	102	help='Variables to process: P, Z, V ,W')
	103	parser.add_argument('--grado', default=2,
	104	help='Angle in Elev to plot')
	105	parser.add_argument('--save', default=0,
	106	help='Save plot')
	107	parser.add_argument('--range', default=0, type=float,
	108	help='Max range to plot')
	109	parser.add_argument('--plot', action='store_true',
	110	help='Create plot files')
	111	parser.add_argument('--show', action='store_true',
	112	help='Show matplotlib plot.')
	113	parser.add_argument('--server', action='store_true',
	114	help='Send to realtime')
	115	args = parser.parse_args()
	116
	117	main(args)

schainpy/model/graphics/jroplot_parameters.py +13 -7

             import os
             import datetime
             import warnings
             import numpy
             from mpl_toolkits.axisartist.grid_finder import FixedLocator, DictFormatter
             from schainpy.model.graphics.jroplot_base import Plot, plt
             from schainpy.model.graphics.jroplot_spectra import SpectraPlot, RTIPlot, CoherencePlot, SpectraCutPlot
             from schainpy.utils import log
             EARTH_RADIUS = 6.3710e3
             def antenna_to_cartesian(ranges, azimuths, elevations):
                 """
                 Return Cartesian coordinates from antenna coordinates.
                 Parameters
                 ----------
                 ranges : array
                     Distances to the center of the radar gates (bins) in kilometers.
                 azimuths : array
                     Azimuth angle of the radar in degrees.
                 elevations : array
                     Elevation angle of the radar in degrees.
                 Returns
                 -------
                 x, y, z : array
                     Cartesian coordinates in meters from the radar.
                 Notes
                 -----
                 The calculation for Cartesian coordinate is adapted from equations
 .28(b) and 2.28(c) of Doviak and Zrnic [1]_ assuming a
                 standard atmosphere (4/3 Earth's radius model).
                 .. math::
                     z = \\sqrt{r^2+R^2+2*r*R*sin(\\theta_e)} - R
                     s = R * arcsin(\\frac{r*cos(\\theta_e)}{R+z})
                     x = s * sin(\\theta_a)
                     y = s * cos(\\theta_a)
                 Where r is the distance from the radar to the center of the gate,
                 :math:`\\theta_a` is the azimuth angle, :math:`\\theta_e` is the
                 elevation angle, s is the arc length, and R is the effective radius
                 of the earth, taken to be 4/3 the mean radius of earth (6371 km).
                 References
                 ----------
                 .. [1] Doviak and Zrnic, Doppler Radar and Weather Observations, Second
                     Edition, 1993, p. 21.
                 """
                 theta_e = numpy.deg2rad(elevations) # elevation angle in radians.
                 theta_a = numpy.deg2rad(azimuths) # azimuth angle in radians.
                 R = 6371.0 * 1000.0 * 4.0 / 3.0     # effective radius of earth in meters.
                 r = ranges * 1000.0                 # distances to gates in meters.
                 z = (r ** 2 + R ** 2 + 2.0 * r * R * numpy.sin(theta_e)) ** 0.5 - R
                 s = R * numpy.arcsin(r * numpy.cos(theta_e) / (R + z))  # arc length in m.
                 x = s * numpy.sin(theta_a)
                 y = s * numpy.cos(theta_a)
                 return x, y, z
             def cartesian_to_geographic_aeqd(x, y, lon_0, lat_0, R=EARTH_RADIUS):
                 """
                 Azimuthal equidistant Cartesian to geographic coordinate transform.
                 Transform a set of Cartesian/Cartographic coordinates (x, y) to
                 geographic coordinate system (lat, lon) using a azimuthal equidistant
                 map projection [1]_.
                 .. math::
                     lat = \\arcsin(\\cos(c) * \\sin(lat_0) +
                                    (y * \\sin(c) * \\cos(lat_0) / \\rho))
                     lon = lon_0 + \\arctan2(
                         x * \\sin(c),
                         \\rho * \\cos(lat_0) * \\cos(c) - y * \\sin(lat_0) * \\sin(c))
                     \\rho = \\sqrt(x^2 + y^2)
                     c = \\rho / R
                 Where x, y are the Cartesian position from the center of projection;
                 lat, lon the corresponding latitude and longitude; lat_0, lon_0 are the
                 latitude and longitude of the center of the projection; R is the radius of
                 the earth (defaults to ~6371 km). lon is adjusted to be between -180 and
 .
                 Parameters
                 ----------
                 x, y : array-like
                     Cartesian coordinates in the same units as R, typically meters.
                 lon_0, lat_0 : float
                     Longitude and latitude, in degrees, of the center of the projection.
                 R : float, optional
                     Earth radius in the same units as x and y. The default value is in
                     units of meters.
                 Returns
                 -------
                 lon, lat : array
                     Longitude and latitude of Cartesian coordinates in degrees.
                 References
                 ----------
                 .. [1] Snyder, J. P. Map Projections--A Working Manual. U. S. Geological
                     Survey Professional Paper 1395, 1987, pp. 191-202.
                 """
                 x = numpy.atleast_1d(numpy.asarray(x))
                 y = numpy.atleast_1d(numpy.asarray(y))
                 lat_0_rad = numpy.deg2rad(lat_0)
                 lon_0_rad = numpy.deg2rad(lon_0)
                 rho = numpy.sqrt(x*x + y*y)
                 c = rho / R
                 with warnings.catch_warnings():
                     # division by zero may occur here but is properly addressed below so
                     # the warnings can be ignored
                     warnings.simplefilter("ignore", RuntimeWarning)
                     lat_rad = numpy.arcsin(numpy.cos(c) * numpy.sin(lat_0_rad) +
                                         y * numpy.sin(c) * numpy.cos(lat_0_rad) / rho)
                 lat_deg = numpy.rad2deg(lat_rad)
                 # fix cases where the distance from the center of the projection is zero
                 lat_deg[rho == 0] = lat_0
                 x1 = x * numpy.sin(c)
                 x2 = rho*numpy.cos(lat_0_rad)*numpy.cos(c) - y*numpy.sin(lat_0_rad)*numpy.sin(c)
                 lon_rad = lon_0_rad + numpy.arctan2(x1, x2)
                 lon_deg = numpy.rad2deg(lon_rad)
                 # Longitudes should be from -180 to 180 degrees
                 lon_deg[lon_deg > 180] -= 360.
                 lon_deg[lon_deg < -180] += 360.
                 return lon_deg, lat_deg
             def antenna_to_geographic(ranges, azimuths, elevations, site):
                 x, y, z = antenna_to_cartesian(numpy.array(ranges), numpy.array(azimuths), numpy.array(elevations))
                 lon, lat = cartesian_to_geographic_aeqd(x, y, site[0], site[1], R=6370997.)
                 return lon, lat
             def ll2xy(lat1, lon1, lat2, lon2):
                 p = 0.017453292519943295
                 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
                     numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
                 r = 12742 * numpy.arcsin(numpy.sqrt(a))
                 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
                                       * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
                 theta = -theta + numpy.pi/2
                 return r*numpy.cos(theta), r*numpy.sin(theta)
             def km2deg(km):
                 '''
                 Convert distance in km to degrees
                 '''
                 return numpy.rad2deg(km/EARTH_RADIUS)
             class SpectralMomentsPlot(SpectraPlot):
                 '''
                 Plot for Spectral Moments
                 '''
                 CODE = 'spc_moments'
                 # colormap = 'jet'
                 # plot_type = 'pcolor'
             class DobleGaussianPlot(SpectraPlot):
                 '''
                 Plot for Double Gaussian Plot
                 '''
                 CODE = 'gaussian_fit'
                 # colormap = 'jet'
                 # plot_type = 'pcolor'
             class DoubleGaussianSpectraCutPlot(SpectraCutPlot):
                 '''
                 Plot SpectraCut with Double Gaussian Fit
                 '''
                 CODE = 'cut_gaussian_fit'
             class SnrPlot(RTIPlot):
                 '''
                 Plot for SNR Data
                 '''
                 CODE = 'snr'
                 colormap = 'jet'
                 def update(self, dataOut):
                     data = {
                         'snr': 10*numpy.log10(dataOut.data_snr)
                     }
                     return data, {}
             class DopplerPlot(RTIPlot):
                 '''
                 Plot for DOPPLER Data (1st moment)
                 '''
                 CODE = 'dop'
                 colormap = 'jet'
                 def update(self, dataOut):
                     data = {
                         'dop': 10*numpy.log10(dataOut.data_dop)
                     }
                     return data, {}
             class PowerPlot(RTIPlot):
                 '''
                 Plot for Power Data (0 moment)
                 '''
                 CODE = 'pow'
                 colormap = 'jet'
                 def update(self, dataOut):
                     data = {
                         'pow': 10*numpy.log10(dataOut.data_pow/dataOut.normFactor)
                     }
                     return data, {}
             class SpectralWidthPlot(RTIPlot):
                 '''
                 Plot for Spectral Width Data (2nd moment)
                 '''
                 CODE = 'width'
                 colormap = 'jet'
                 def update(self, dataOut):
                     data = {
                         'width': dataOut.data_width
                     }
                     return data, {}
             class SkyMapPlot(Plot):
                 '''
                 Plot for meteors detection data
                 '''
                 CODE = 'param'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = 1
                     self.width = 7.2
                     self.height = 7.2
                     self.nplots = 1
                     self.xlabel = 'Zonal Zenith Angle (deg)'
                     self.ylabel = 'Meridional Zenith Angle (deg)'
                     self.polar = True
                     self.ymin = -180
                     self.ymax = 180
                     self.colorbar = False
                 def plot(self):
                     arrayParameters = numpy.concatenate(self.data['param'])
                     error = arrayParameters[:, -1]
                     indValid = numpy.where(error == 0)[0]
                     finalMeteor = arrayParameters[indValid, :]
                     finalAzimuth = finalMeteor[:, 3]
                     finalZenith = finalMeteor[:, 4]
                     x = finalAzimuth * numpy.pi / 180
                     y = finalZenith
                     ax = self.axes[0]
                     if ax.firsttime:
                         ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
                     else:
                         ax.plot.set_data(x, y)
                     dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
                     dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
                     title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
                                                                                                  dt2,
                                                                                                  len(x))
                     self.titles[0] = title
             class GenericRTIPlot(Plot):
                 '''
                 Plot for data_xxxx object
                 '''
                 CODE = 'param'
                 colormap = 'viridis'
                 plot_type = 'pcolorbuffer'
                 def setup(self):
                     self.xaxis = 'time'
                     self.ncols = 1
                     self.nrows = self.data.shape('param')[0]
                     self.nplots = self.nrows
                     self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
                     if not self.xlabel:
                         self.xlabel = 'Time'
                     self.ylabel = 'Range [km]'
                     if not self.titles:
                         self.titles = ['Param {}'.format(x) for x in range(self.nrows)]
                 def update(self, dataOut):
                     data = {
                         'param' : numpy.concatenate([getattr(dataOut, attr) for attr in self.attr_data], axis=0)
                     }
                     meta = {}
                     return data, meta
                 def plot(self):
                     # self.data.normalize_heights()
                     self.x = self.data.times
                     self.y = self.data.yrange
                     self.z = self.data['param']
                     self.z = 10*numpy.log10(self.z)
                     self.z = numpy.ma.masked_invalid(self.z)
                     if self.decimation is None:
                         x, y, z = self.fill_gaps(self.x, self.y, self.z)
                     else:
                         x, y, z = self.fill_gaps(*self.decimate())
                     for n, ax in enumerate(self.axes):
                         self.zmax = self.zmax if self.zmax is not None else numpy.max(
                             self.z[n])
                         self.zmin = self.zmin if self.zmin is not None else numpy.min(
                             self.z[n])
                         if ax.firsttime:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=self.cmaps[n]
                                                    )
                         else:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.collections.remove(ax.collections[0])
                             ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
                                                    vmin=self.zmin,
                                                    vmax=self.zmax,
                                                    cmap=self.cmaps[n]
                                                    )
             class PolarMapPlot(Plot):
                 '''
                 Plot for weather radar
                 '''
                 CODE = 'param'
                 colormap = 'seismic'
                 def setup(self):
                     self.ncols = 1
                     self.nrows = 1
                     self.width = 9
                     self.height = 8
                     self.mode = self.data.meta['mode']
                     if self.channels is not None:
                         self.nplots = len(self.channels)
                         self.nrows = len(self.channels)
                     else:
                         self.nplots = self.data.shape(self.CODE)[0]
                         self.nrows = self.nplots
                         self.channels = list(range(self.nplots))
                     if self.mode == 'E':
                         self.xlabel = 'Longitude'
                         self.ylabel = 'Latitude'
                     else:
                         self.xlabel = 'Range (km)'
                         self.ylabel = 'Height (km)'
                     self.bgcolor = 'white'
                     self.cb_labels = self.data.meta['units']
                     self.lat = self.data.meta['latitude']
                     self.lon = self.data.meta['longitude']
                     self.xmin, self.xmax = float(
                         km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
                     self.ymin, self.ymax = float(
                         km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
                     #  self.polar = True
                 def plot(self):
                     for n, ax in enumerate(self.axes):
                         data = self.data['param'][self.channels[n]]
                         zeniths = numpy.linspace(
 , self.data.meta['max_range'], data.shape[1])
                         if self.mode == 'E':
                             azimuths = -numpy.radians(self.data.yrange)+numpy.pi/2
                             r, theta = numpy.meshgrid(zeniths, azimuths)
                             x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
                                 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
                             x = km2deg(x) + self.lon
                             y = km2deg(y) + self.lat
                         else:
                             azimuths = numpy.radians(self.data.yrange)
                             r, theta = numpy.meshgrid(zeniths, azimuths)
                             x, y = r*numpy.cos(theta), r*numpy.sin(theta)
                         self.y = zeniths
                         if ax.firsttime:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.plt = ax.pcolormesh(  # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
                                 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
                                 vmin=self.zmin,
                                 vmax=self.zmax,
                                 cmap=self.cmaps[n])
                         else:
                             if self.zlimits is not None:
                                 self.zmin, self.zmax = self.zlimits[n]
                             ax.collections.remove(ax.collections[0])
                             ax.plt = ax.pcolormesh(  # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
                                 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
                                 vmin=self.zmin,
                                 vmax=self.zmax,
                                 cmap=self.cmaps[n])
                         if self.mode == 'A':
                             continue
                         # plot district names
                         f = open('/data/workspace/schain_scripts/distrito.csv')
                         for line in f:
                             label, lon, lat = [s.strip() for s in line.split(',') if s]
                             lat = float(lat)
                             lon = float(lon)
                             # ax.plot(lon, lat, '.b', ms=2)
                             ax.text(lon, lat, label.decode('utf8'), ha='center',
                                     va='bottom', size='8', color='black')
                         # plot limites
                         limites = []
                         tmp = []
                         for line in open('/data/workspace/schain_scripts/lima.csv'):
                             if '#' in line:
                                 if tmp:
                                     limites.append(tmp)
                                 tmp = []
                                 continue
                             values = line.strip().split(',')
                             tmp.append((float(values[0]), float(values[1])))
                         for points in limites:
                             ax.add_patch(
                                 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
                         # plot Cuencas
                         for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
                             f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
                             values = [line.strip().split(',') for line in f]
                             points = [(float(s[0]), float(s[1])) for s in values]
                             ax.add_patch(Polygon(points, ec='b', fc='none'))
                         # plot grid
                         for r in (15, 30, 45, 60):
                             ax.add_artist(plt.Circle((self.lon, self.lat),
                                                      km2deg(r), color='0.6', fill=False, lw=0.2))
                             ax.text(
                                 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
                                 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
                                 '{}km'.format(r),
                                 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
                     if self.mode == 'E':
                         title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
                         label = 'E{:02d}'.format(int(self.data.meta['elevation']))
                     else:
                         title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
                         label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
                     self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
                     self.titles = ['{} {}'.format(
                         self.data.parameters[x], title) for x in self.channels]
             class WeatherParamsPlot(Plot):
                 #CODE = 'RHI'
                 #plot_name = 'RHI'
                 plot_type = 'scattermap'
                 buffering = False
                 def setup(self):
                     self.ncols = 1
                     self.nrows = 1
                     self.nplots= 1
                     self.ylabel= 'Range [km]'
                     self.xlabel= 'Range [km]'
                     self.polar = True
                     self.grid = True
                     if self.channels is not None:
                         self.nplots = len(self.channels)
                         self.nrows = len(self.channels)
                     else:
                         self.nplots = self.data.shape(self.CODE)[0]
                         self.nrows = self.nplots
                         self.channels = list(range(self.nplots))
                     self.colorbar=True
                     self.width   =8
                     self.height  =8
                     self.ini     =0
                     self.len_azi =0
                     self.buffer_ini  = None
                     self.buffer_ele   = None
                     self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08})
                     self.flag    =0
                     self.indicador= 0
                     self.last_data_ele = None
                     self.val_mean      = None
                 def update(self, dataOut):
                     vars = {
                         'S' : 0,
                         'V' : 1,
                         'W' : 2,
                         'SNR' : 3,
                         'Z' : 4,
                         'D' : 5,
                         'P' : 6,
                         'R' : 7,
                     }
                     data = {}
                     meta = {}
                     if hasattr(dataOut, 'nFFTPoints'):
                         factor = dataOut.normFactor
                     else:
                         factor = 1
-                    if 'S' in self.attr_data[0]:
+                    if hasattr(dataOut, 'dparam'):
-                        tmp = 10*numpy.log10(10.0*getattr(dataOut, 'data_param')[:,0,:]/(factor))
+                        tmp = getattr(dataOut, 'data_param')
                     else:
-                        tmp = getattr(dataOut, 'data_param')[:,vars[self.attr_data[0]],:]
+                        if 'S' in self.attr_data[0]:
+                            tmp = 10*numpy.log10(10.0*getattr(dataOut, 'data_param')[:,0,:]/(factor))
+                        else:
+                            tmp = getattr(dataOut, 'data_param')[:,vars[self.attr_data[0]],:]
                     if self.mask:
                         mask = dataOut.data_param[:,3,:] < self.mask
                         tmp = numpy.ma.masked_array(tmp, mask=mask)
                     r = dataOut.heightList
                     delta_height = r[1]-r[0]
                     valid = numpy.where(r>=0)[0]
                     data['r'] = numpy.arange(len(valid))*delta_height
                     try:
                         data['data'] = tmp[self.channels[0]][:,valid]
                     except:
                         data['data'] = tmp[0][:,valid]
                     if dataOut.mode_op == 'PPI':
                         self.CODE = 'PPI'
                         self.title = self.CODE
                     elif dataOut.mode_op == 'RHI':
                         self.CODE = 'RHI'
                         self.title = self.CODE
                     data['azi'] = dataOut.data_azi
                     data['ele'] = dataOut.data_ele
                     data['mode_op'] = dataOut.mode_op
                     self.mode = dataOut.mode_op
                     var = data['data'].flatten()
                     r = numpy.tile(data['r'], data['data'].shape[0])
                     az = numpy.repeat(data['azi'], data['data'].shape[1])
                     el = numpy.repeat(data['ele'], data['data'].shape[1])
                     # lla = georef.spherical_to_proj(r, data['azi'], data['ele'], (-75.295893, -12.040436, 3379.2147))
                     latlon = antenna_to_geographic(r, az, el, (-75.295893, -12.040436))
                     if self.mask:
                         meta['lat'] = latlon[1][var.mask==False]
                         meta['lon'] = latlon[0][var.mask==False]
                         data['var'] = numpy.array([var[var.mask==False]])
                     else:
                         meta['lat'] = latlon[1]
                         meta['lon'] = latlon[0]
                         data['var'] = numpy.array([var])
                     return data, meta
                 def plot(self):
                     data = self.data[-1]
                     z = data['data']
                     r = data['r']
                     self.titles = []
                     self.ymax = self.ymax if self.ymax else numpy.nanmax(r)
                     self.ymin = self.ymin if self.ymin else numpy.nanmin(r)
                     self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
                     self.zmin = self.zmin if self.zmin is not None else numpy.nanmin(z)
+                    if isinstance(data['mode_op'], bytes):
+                        data['mode_op'] = data['mode_op'].decode()
                     if data['mode_op'] == 'RHI':
                         try:
                             if self.data['mode_op'][-2] == 'PPI':
                                 self.ang_min = None
                                 self.ang_max = None
                         except:
                             pass
                         self.ang_min = self.ang_min if self.ang_min else 0
                         self.ang_max = self.ang_max if self.ang_max else 90
                         r, theta = numpy.meshgrid(r, numpy.radians(data['ele']) )
                     elif data['mode_op'] == 'PPI':
                         try:
                             if self.data['mode_op'][-2] == 'RHI':
                                 self.ang_min = None
                                 self.ang_max = None
                         except:
                             pass
                         self.ang_min = self.ang_min if self.ang_min else 0
                         self.ang_max = self.ang_max if self.ang_max else 360
                         r, theta = numpy.meshgrid(r, numpy.radians(data['azi']) )
                     self.clear_figures()
                     for i,ax in enumerate(self.axes):
                         if ax.firsttime:
                             ax.set_xlim(numpy.radians(self.ang_min),numpy.radians(self.ang_max))
                             ax.plt = ax.pcolormesh(theta, r, z, cmap=self.colormap, vmin=self.zmin, vmax=self.zmax)
                             if data['mode_op'] == 'PPI':
                                 ax.set_theta_direction(-1)
                                 ax.set_theta_offset(numpy.pi/2)
                         else:
                             ax.set_xlim(numpy.radians(self.ang_min),numpy.radians(self.ang_max))
                             ax.plt = ax.pcolormesh(theta, r, z, cmap=self.colormap, vmin=self.zmin, vmax=self.zmax)
                             if data['mode_op'] == 'PPI':
                                 ax.set_theta_direction(-1)
                                 ax.set_theta_offset(numpy.pi/2)
                         ax.grid(True)
                         if data['mode_op'] == 'RHI':
                             len_aux = int(data['azi'].shape[0]/4)
                             mean = numpy.mean(data['azi'][len_aux:-len_aux])
                             if len(self.channels) !=1:
                                 self.titles = ['RHI {} at AZ: {} CH {}'.format(self.labels[x], str(round(mean,1)), x) for x in range(self.nrows)]
                             else:
                                 self.titles = ['RHI {} at AZ: {} CH {}'.format(self.labels[0], str(round(mean,1)), self.channels[0])]
                         elif data['mode_op'] == 'PPI':
                             len_aux = int(data['ele'].shape[0]/4)
                             mean = numpy.mean(data['ele'][len_aux:-len_aux])
                             if len(self.channels) !=1:
                                 self.titles = ['PPI {} at EL: {} CH {}'.format(self.labels[x], str(round(mean,1)), x) for x in range(self.nrows)]
                             else:
                                 self.titles = ['PPI {} at EL: {} CH {}'.format(self.labels[0], str(round(mean,1)), self.channels[0])]
                         self.mode_value = round(mean,1)

schainpy/model/io/jroIO_base.py +8 -6

             """
             Created on Jul 2, 2014
             @author: roj-idl71
             """
             import os
             import sys
             import glob
             import time
             import numpy
             import fnmatch
             import inspect
             import time
             import datetime
             import zmq
             from schainpy.model.proc.jroproc_base import Operation, MPDecorator
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
             from schainpy.utils import log
             import schainpy.admin
             LOCALTIME = True
             DT_DIRECTIVES = {
                 '%Y': 4,
                 '%y': 2,
                 '%m': 2,
                 '%d': 2,
                 '%j': 3,
                 '%H': 2,
                 '%M': 2,
                 '%S': 2,
                 '%f': 6
             }
             def isNumber(cad):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float(cad)
                     return True
                 except:
                     return False
             def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
                 """
                 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startUTSeconds      :    fecha inicial del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                     endUTSeconds        :    fecha final del rango seleccionado. La fecha esta dada en
                                              segundos contados desde 01/01/1970.
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 basicHeaderObj = BasicHeader(LOCALTIME)
                 try:
                     fp = open(filename, 'rb')
                 except IOError:
                     print("The file %s can't be opened" % (filename))
                     return 0
                 sts = basicHeaderObj.read(fp)
                 fp.close()
                 if not(sts):
                     print("Skipping the file %s because it has not a valid header" % (filename))
                     return 0
                 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
                     return 0
                 return 1
             def isTimeInRange(thisTime, startTime, endTime):
                 if endTime >= startTime:
                     if (thisTime < startTime) or (thisTime > endTime):
                         return 0
                     return 1
                 else:
                     if (thisTime < startTime) and (thisTime > endTime):
                         return 0
                     return 1
             def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                     startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                     endTime            :    tiempo final del rango seleccionado en formato datetime.time
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no existe o no puede ser abierto
                     Si la cabecera no puede ser leida.
                 """
                 try:
                     fp = open(filename, 'rb')
                 except IOError:
                     print("The file %s can't be opened" % (filename))
                     return None
                 firstBasicHeaderObj = BasicHeader(LOCALTIME)
                 systemHeaderObj = SystemHeader()
                 radarControllerHeaderObj = RadarControllerHeader()
                 processingHeaderObj = ProcessingHeader()
                 lastBasicHeaderObj = BasicHeader(LOCALTIME)
                 sts = firstBasicHeaderObj.read(fp)
                 if not(sts):
                     print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
                     return None
                 if not systemHeaderObj.read(fp):
                     return None
                 if not radarControllerHeaderObj.read(fp):
                     return None
                 if not processingHeaderObj.read(fp):
                     return None
                 filesize = os.path.getsize(filename)
                 offset = processingHeaderObj.blockSize + 24  # header size
                 if filesize <= offset:
                     print("[Reading] %s: This file has not enough data" % filename)
                     return None
                 fp.seek(-offset, 2)
                 sts = lastBasicHeaderObj.read(fp)
                 fp.close()
                 thisDatetime = lastBasicHeaderObj.datatime
                 thisTime_last_block = thisDatetime.time()
                 thisDatetime = firstBasicHeaderObj.datatime
                 thisDate = thisDatetime.date()
                 thisTime_first_block = thisDatetime.time()
                 # General case
                 #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                 #-----------o----------------------------o-----------
                 #       startTime                     endTime
                 if endTime >= startTime:
                     if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
                         return None
                     return thisDatetime
                 # If endTime < startTime then endTime belongs to the next day
                 #<<<<<<<<<<<o                            o>>>>>>>>>>>
                 #-----------o----------------------------o-----------
                 #        endTime                    startTime
                 if (thisDate == startDate) and (thisTime_last_block < startTime):
                     return None
                 if (thisDate == endDate) and (thisTime_first_block > endTime):
                     return None
                 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
                     return None
                 return thisDatetime
             def isFolderInDateRange(folder, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     folder            :    nombre completo del directorio.
                                            Su formato deberia ser "/path_root/?YYYYDDD"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el directorio no tiene el formato adecuado
                 """
                 basename = os.path.basename(folder)
                 if not isRadarFolder(basename):
                     print("The folder %s has not the rigth format" % folder)
                     return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFolder(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def isFileInDateRange(filename, startDate=None, endDate=None):
                 """
                 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                 Inputs:
                     filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                                            Su formato deberia ser "?YYYYDDDsss"
                                             siendo:
                                                 YYYY    :    Anio         (ejemplo 2015)
                                                 DDD     :    Dia del anio (ejemplo 305)
                                                 sss     :    set
                     startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                     endDate            :    fecha final del rango seleccionado en formato datetime.date
                 Return:
                     Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                     fecha especificado, de lo contrario retorna False.
                 Excepciones:
                     Si el archivo no tiene el formato adecuado
                 """
                 basename = os.path.basename(filename)
                 if not isRadarFile(basename):
                     print("The filename %s has not the rigth format" % filename)
                     return 0
                 if startDate and endDate:
                     thisDate = getDateFromRadarFile(basename)
                     if thisDate < startDate:
                         return 0
                     if thisDate > endDate:
                         return 0
                 return 1
             def getFileFromSet(path, ext, set):
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     try:
                         year = int(thisFile[1:5])
                         doy = int(thisFile[5:8])
                     except:
                         continue
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 myfile = fnmatch.filter(
                     validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
                 if len(myfile) != 0:
                     return myfile[0]
                 else:
                     filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
                     print('the filename %s does not exist' % filename)
                     print('...going to the last file: ')
                 if validFilelist:
                     validFilelist = sorted(validFilelist, key=str.lower)
                     return validFilelist[-1]
                 return None
             def getlastFileFromPath(path, ext):
                 """
                 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     year = thisFile[1:5]
                     if not isNumber(year):
                         continue
                     doy = thisFile[5:8]
                     if not isNumber(doy):
                         continue
                     year = int(year)
                     doy = int(doy)
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 if validFilelist:
                     validFilelist = sorted(validFilelist, key=str.lower)
                     return validFilelist[-1]
                 return None
             def isRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return 0
                 return 1
             def isRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
                 except:
                     return 0
                 return 1
             def getDateFromRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
                 return thisDate
             def getDateFromRadarFolder(folder):
                 try:
                     year = int(folder[1:5])
                     doy = int(folder[5:8])
                 except:
                     return None
                 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
                 return thisDate
             def parse_format(s, fmt):
                 for i in range(fmt.count('%')):
                     x = fmt.index('%')
                     d = DT_DIRECTIVES[fmt[x:x+2]]
                     fmt = fmt.replace(fmt[x:x+2], s[x:x+d])
                 return fmt
             class Reader(object):
                 c = 3E8
                 isConfig = False
                 dtype = None
                 pathList = []
                 filenameList = []
                 datetimeList = []
                 filename = None
                 ext = None
                 flagIsNewFile = 1
                 flagDiscontinuousBlock = 0
                 flagIsNewBlock = 0
                 flagNoMoreFiles = 0
                 fp = None
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 versionFile = 1103
                 fileSize = None
                 fileSizeByHeader = None
                 fileIndex = -1
                 profileIndex = None
                 blockIndex = 0
                 nTotalBlocks = 0
                 maxTimeStep = 30
                 lastUTTime = None
                 datablock = None
                 dataOut = None
                 getByBlock = False
                 path = None
                 startDate = None
                 endDate = None
                 startTime = datetime.time(0, 0, 0)
                 endTime = datetime.time(23, 59, 59)
                 set = None
                 expLabel = ""
                 online = False
                 delay = 60
                 nTries = 3  # quantity tries
                 nFiles = 3  # number of files for searching
                 walk = True
                 getblock = False
                 nTxs = 1
                 realtime = False
                 blocksize = 0
                 blocktime = None
                 warnings = True
                 verbose = True
                 server = None
                 format = None
                 oneDDict = None
                 twoDDict = None
                 independentParam = None
                 filefmt = None
                 folderfmt = None
                 open_file = open
                 open_mode = 'rb'
+                filter =None
                 def run(self):
                     raise NotImplementedError
                 def getAllowedArgs(self):
                     if hasattr(self, '__attrs__'):
                         return self.__attrs__
                     else:
                         return inspect.getargspec(self.run).args
                 def set_kwargs(self, **kwargs):
                     for key, value in kwargs.items():
                         setattr(self, key, value)
                 def find_folders(self, path, startDate, endDate, folderfmt, last=False):
                     folders = [x for f in path.split(',')
                                for x in os.listdir(f) if os.path.isdir(os.path.join(f, x))]
                     folders.sort()
                     if last:
                         folders = [folders[-1]]
                     for folder in folders:
                         try:
                             dt = datetime.datetime.strptime(parse_format(folder, folderfmt), folderfmt).date()
                             if dt >= startDate and dt <= endDate:
                                 yield os.path.join(path, folder)
                             else:
                                 log.log('Skiping folder {}'.format(folder), self.name)
                         except Exception as e:
                             log.log('Skiping folder {}'.format(folder), self.name)
                             continue
                     return
                 def find_files(self, folders, ext, filefmt, startDate=None, endDate=None,
-                               expLabel='', last=False):
+                               expLabel='', filter=None,last=False):
                     for path in folders:
                         files = glob.glob1(path, '*{}'.format(ext))
                         files.sort()
+                        if filter is not None:
+                            files= [ file for file in files if  os.path.splitext(file)[0][-len(filter):] == filter]
                         if last:
                             if files:
                                 fo = files[-1]
                                 try:
                                     dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
                                     yield os.path.join(path, expLabel, fo)
                                 except Exception as e:
                                     pass
                                 return
                             else:
                                 return
                         for fo in files:
                             try:
                                 dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
                                 if dt >= startDate and dt <= endDate:
                                     yield os.path.join(path, expLabel, fo)
                                 else:
                                     log.log('Skiping file {}'.format(fo), self.name)
                             except Exception as e:
                                 log.log('Skiping file {}'.format(fo), self.name)
                                 continue
                 def searchFilesOffLine(self, path, startDate, endDate,
                                        expLabel, ext, walk,
-                                       filefmt, folderfmt):
+                                       filefmt, folderfmt,filter):
                     """Search files in offline mode for the given arguments
                     Return:
                         Generator of files
                     """
                     if walk:
                         folders = self.find_folders(
                             path, startDate, endDate, folderfmt)
                     else:
                         folders = path.split(',')
                     return self.find_files(
-                        folders, ext, filefmt, startDate, endDate, expLabel)
+                        folders, ext, filefmt, startDate, endDate, expLabel,filter)
                 def searchFilesOnLine(self, path, startDate, endDate,
                                       expLabel, ext, walk,
-                                      filefmt, folderfmt):
+                                      filefmt, folderfmt,filter):
                     """Search for the last file of the last folder
                     Arguments:
                         path        :    carpeta donde estan contenidos los files que contiene data
                         expLabel    :    Nombre del subexperimento (subfolder)
                         ext         :    extension de los files
                         walk        :    Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
                     Return:
                         generator with the full path of last filename
                     """
                     if walk:
                         folders = self.find_folders(
                             path, startDate, endDate, folderfmt, last=True)
                     else:
                         folders = path.split(',')
-                    return self.find_files(
+                    return self.find_files(folders, ext, filefmt, startDate, endDate, expLabel, filter,last=True)
-                        folders, ext, filefmt, startDate, endDate, expLabel, last=True)
                 def setNextFile(self):
                     """Set the next file to be readed open it and parse de file header"""
                     while True:
                         if self.fp != None:
                             self.fp.close()
                         if self.online:
                             newFile = self.setNextFileOnline()
                         else:
                             newFile = self.setNextFileOffline()
                         if not(newFile):
                             if self.online:
                                 raise schainpy.admin.SchainError('Time to wait for new files reach')
                             else:
                                 if self.fileIndex == -1:
                                     raise schainpy.admin.SchainWarning('No files found in the given path')
                                 else:
                                     raise schainpy.admin.SchainWarning('No more files to read')
                         if self.verifyFile(self.filename):
                             break
                     log.log('Opening file: %s' % self.filename, self.name)
                     self.readFirstHeader()
                     self.nReadBlocks = 0
                 def setNextFileOnline(self):
                     """Check for the next file to be readed in online mode.
                     Set:
                         self.filename
                         self.fp
                         self.filesize
                     Return:
                         boolean
                     """
                     nextFile = True
                     nextDay = False
                     for nFiles in range(self.nFiles+1):
                         for nTries in range(self.nTries):
                             fullfilename, filename = self.checkForRealPath(nextFile, nextDay)
                             if fullfilename is not None:
                                 break
                             log.warning(
                                 "Waiting %0.2f sec for the next file: \"%s\" , try %02d ..." % (self.delay, filename, nTries + 1),
                                 self.name)
                             time.sleep(self.delay)
                             nextFile = False
                             continue
                         if fullfilename is not None:
                             break
                         self.nTries = 1
                         nextFile = True
                         if nFiles == (self.nFiles - 1):
                             log.log('Trying with next day...', self.name)
                             nextDay = True
                             self.nTries = 3
                     if fullfilename:
                         self.fileSize = os.path.getsize(fullfilename)
                         self.filename = fullfilename
                         self.flagIsNewFile = 1
                         if self.fp != None:
                             self.fp.close()
                         self.fp = self.open_file(fullfilename, self.open_mode)
                         self.flagNoMoreFiles = 0
                         self.fileIndex += 1
                         return 1
                     else:
                         return 0
                 def setNextFileOffline(self):
                     """Open the next file to be readed in offline mode"""
                     try:
                         filename = next(self.filenameList)
                         self.fileIndex +=1
                     except StopIteration:
                         self.flagNoMoreFiles = 1
                         return 0
                     self.filename = filename
                     self.fileSize = os.path.getsize(filename)
                     self.fp = self.open_file(filename, self.open_mode)
                     self.flagIsNewFile = 1
                     return 1
                 @staticmethod
                 def isDateTimeInRange(dt, startDate, endDate, startTime, endTime):
                     """Check if the given datetime is in range"""
                     if startDate <= dt.date() <= endDate:
                         if startTime <= dt.time() <= endTime:
                             return True
                     return False
                 def verifyFile(self, filename):
                     """Check for a valid file
                     Arguments:
                         filename -- full path filename
                     Return:
                         boolean
                     """
                     return True
                 def checkForRealPath(self, nextFile, nextDay):
                     """Check if the next file to be readed exists"""
                     if nextFile:
                         self.set += 1
                     if nextDay:
                         self.set = 0
                         self.doy += 1
                     foldercounter = 0
                     prefixDirList = [None, 'd', 'D']
                     if self.ext.lower() == ".r":  # voltage
                         prefixFileList = ['d', 'D']
                     elif self.ext.lower() == ".pdata":  # spectra
                         prefixFileList = ['p', 'P']
                     elif self.ext.lower() == ".hdf5":  # HDF5
                         prefixFileList = ['D', 'P']    # HDF5
                     # barrido por las combinaciones posibles
                     for prefixDir in prefixDirList:
                         thispath = self.path
                         if prefixDir != None:
                             # formo el nombre del directorio xYYYYDDD (x=d o x=D)
                             if foldercounter == 0:
                                 thispath = os.path.join(self.path, "%s%04d%03d" %
                                                         (prefixDir, self.year, self.doy))
                             else:
                                 thispath = os.path.join(self.path, "%s%04d%03d_%02d" % (
                                     prefixDir, self.year, self.doy, foldercounter))
                         for prefixFile in prefixFileList:  # barrido por las dos combinaciones posibles de "D"
                             # formo el nombre del file xYYYYDDDSSS.ext
                             filename = "%s%04d%03d%03d%s" % (prefixFile, self.year, self.doy, self.set, self.ext)
                             fullfilename = os.path.join(
                                 thispath, filename)
                             if os.path.exists(fullfilename):
                                 return fullfilename, filename
                     return None, filename
                     #raise NotImplementedError
                 def readFirstHeader(self):
                     """Parse the file header"""
                     pass
                 def waitDataBlock(self, pointer_location, blocksize=None):
                     """
                     """
                     currentPointer = pointer_location
                     if blocksize is None:
                         neededSize = self.processingHeaderObj.blockSize  # + self.basicHeaderSize
                     else:
                         neededSize = blocksize
                     for nTries in range(self.nTries):
                         self.fp.close()
                         self.fp = open(self.filename, 'rb')
                         self.fp.seek(currentPointer)
                         self.fileSize = os.path.getsize(self.filename)
                         currentSize = self.fileSize - currentPointer
                         if (currentSize >= neededSize):
                             return 1
                         log.warning(
                             "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
                             self.name
                             )
                         time.sleep(self.delay)
                     return 0
             class JRODataReader(Reader):
                 utc = 0
                 nReadBlocks = 0
                 foldercounter = 0
                 firstHeaderSize = 0
                 basicHeaderSize = 24
                 __isFirstTimeOnline = 1
                 filefmt = "*%Y%j***"
                 folderfmt = "*%Y%j"
                 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'online', 'delay', 'walk']
                 def getDtypeWidth(self):
                     dtype_index = get_dtype_index(self.dtype)
                     dtype_width = get_dtype_width(dtype_index)
                     return dtype_width
                 def checkForRealPath(self, nextFile, nextDay):
                     """Check if the next file to be readed exists.
                     Example    :
                         nombre correcto del file es  .../.../D2009307/P2009307367.ext
                         Entonces la funcion prueba con las siguientes combinaciones
                             .../.../y2009307367.ext
                             .../.../Y2009307367.ext
                             .../.../x2009307/y2009307367.ext
                             .../.../x2009307/Y2009307367.ext
                             .../.../X2009307/y2009307367.ext
                             .../.../X2009307/Y2009307367.ext
                         siendo para este caso, la ultima combinacion de letras, identica al file buscado
                     Return:
                         str -- fullpath of the file
                     """
                     if nextFile:
                         self.set += 1
                     if nextDay:
                         self.set = 0
                         self.doy += 1
                     foldercounter = 0
                     prefixDirList = [None, 'd', 'D']
                     if self.ext.lower() == ".r":  # voltage
                         prefixFileList = ['d', 'D']
                     elif self.ext.lower() == ".pdata":  # spectra
                         prefixFileList = ['p', 'P']
                     # barrido por las combinaciones posibles
                     for prefixDir in prefixDirList:
                         thispath = self.path
                         if prefixDir != None:
                             # formo el nombre del directorio xYYYYDDD (x=d o x=D)
                             if foldercounter == 0:
                                 thispath = os.path.join(self.path, "%s%04d%03d" %
                                                         (prefixDir, self.year, self.doy))
                             else:
                                 thispath = os.path.join(self.path, "%s%04d%03d_%02d" % (
                                     prefixDir, self.year, self.doy, foldercounter))
                         for prefixFile in prefixFileList:  # barrido por las dos combinaciones posibles de "D"
                             # formo el nombre del file xYYYYDDDSSS.ext
                             filename = "%s%04d%03d%03d%s" % (prefixFile, self.year, self.doy, self.set, self.ext)
                             fullfilename = os.path.join(
                                 thispath, filename)
                             if os.path.exists(fullfilename):
                                 return fullfilename, filename
                     return None, filename
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range(self.nTries):
                         self.fp.close()
                         self.fp = open(self.filename, 'rb')
                         self.fp.seek(currentPointer)
                         self.fileSize = os.path.getsize(self.filename)
                         currentSize = self.fileSize - currentPointer
                         if (currentSize >= neededSize):
                             self.basicHeaderObj.read(self.fp)
                             return 1
                         if self.fileSize == self.fileSizeByHeader:
                             #                self.flagEoF = True
                             return 0
                         print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
                         time.sleep(self.delay)
                     return 0
                 def __setNewBlock(self):
                     if self.fp == None:
                         return 0
                     if self.flagIsNewFile:
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if self.realtime:
                         self.flagDiscontinuousBlock = 1
                         if not(self.setNextFile()):
                             return 0
                         else:
                             return 1
                     currentSize = self.fileSize - self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     if (currentSize >= neededSize):
                         self.basicHeaderObj.read(self.fp)
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if self.__waitNewBlock():
                         self.lastUTTime = self.basicHeaderObj.utc
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.basicHeaderObj.utc - self.lastUTTime
                     self.lastUTTime = self.basicHeaderObj.utc
                     self.flagDiscontinuousBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagDiscontinuousBlock = 1
                     return 1
                 def readNextBlock(self):
                     while True:
                         if not(self.__setNewBlock()):
                             continue
                         if not(self.readBlock()):
                             return 0
                         self.getBasicHeader()
                         if not self.isDateTimeInRange(self.dataOut.datatime, self.startDate, self.endDate, self.startTime, self.endTime):
                             print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
                                                                                   self.processingHeaderObj.dataBlocksPerFile,
                                                                                   self.dataOut.datatime.ctime()))
                             continue
                         break
                     if self.verbose:
                         print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
                                                                    self.processingHeaderObj.dataBlocksPerFile,
                                                                    self.dataOut.datatime.ctime()))
                     return 1
                 def readFirstHeader(self):
                     self.basicHeaderObj.read(self.fp)
                     self.systemHeaderObj.read(self.fp)
                     self.radarControllerHeaderObj.read(self.fp)
                     self.processingHeaderObj.read(self.fp)
                     self.firstHeaderSize = self.basicHeaderObj.size
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
                                                PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
                     else:
                         raise ValueError('Data type was not defined')
                     self.dtype = datatype_str
                     #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
                     self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
                         self.firstHeaderSize + self.basicHeaderSize * \
                         (self.processingHeaderObj.dataBlocksPerFile - 1)
                     #        self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
                     #        self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
                     self.getBlockDimension()
                 def verifyFile(self, filename):
                     flag = True
                     try:
                         fp = open(filename, 'rb')
                     except IOError:
                         log.error("File {} can't be opened".format(filename), self.name)
                         return False
                     if self.online and self.waitDataBlock(0):
                         pass
                     basicHeaderObj = BasicHeader(LOCALTIME)
                     systemHeaderObj = SystemHeader()
                     radarControllerHeaderObj = RadarControllerHeader()
                     processingHeaderObj = ProcessingHeader()
                     if not(basicHeaderObj.read(fp)):
                         flag = False
                     if not(systemHeaderObj.read(fp)):
                         flag = False
                     if not(radarControllerHeaderObj.read(fp)):
                         flag = False
                     if not(processingHeaderObj.read(fp)):
                         flag = False
                     if not self.online:
                         dt1 = basicHeaderObj.datatime
                         pos = self.fileSize-processingHeaderObj.blockSize-24
                         if pos<0:
                             flag = False
                             log.error('Invalid size for file: {}'.format(self.filename), self.name)
                         else:
                             fp.seek(pos)
                             if not(basicHeaderObj.read(fp)):
                                 flag = False
                         dt2 = basicHeaderObj.datatime
                         if not self.isDateTimeInRange(dt1, self.startDate, self.endDate, self.startTime, self.endTime) and not \
                                 self.isDateTimeInRange(dt2, self.startDate, self.endDate, self.startTime, self.endTime):
                             flag = False
                     fp.close()
                     return flag
                 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
                     path_empty = True
                     dateList = []
                     pathList = []
                     multi_path = path.split(',')
                     if not walk:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             fileList = glob.glob1(single_path, "*" + ext)
                             if not fileList:
                                 continue
                             path_empty = False
                             fileList.sort()
                             for thisFile in fileList:
                                 if not os.path.isfile(os.path.join(single_path, thisFile)):
                                     continue
                                 if not isRadarFile(thisFile):
                                     continue
                                 if not isFileInDateRange(thisFile, startDate, endDate):
                                     continue
                                 thisDate = getDateFromRadarFile(thisFile)
                                 if thisDate in dateList or single_path in pathList:
                                     continue
                                 dateList.append(thisDate)
                                 pathList.append(single_path)
                     else:
                         for single_path in multi_path:
                             if not os.path.isdir(single_path):
                                 continue
                             dirList = []
                             for thisPath in os.listdir(single_path):
                                 if not os.path.isdir(os.path.join(single_path, thisPath)):
                                     continue
                                 if not isRadarFolder(thisPath):
                                     continue
                                 if not isFolderInDateRange(thisPath, startDate, endDate):
                                     continue
                                 dirList.append(thisPath)
                             if not dirList:
                                 continue
                             dirList.sort()
                             for thisDir in dirList:
                                 datapath = os.path.join(single_path, thisDir, expLabel)
                                 fileList = glob.glob1(datapath, "*" + ext)
                                 if not fileList:
                                     continue
                                 path_empty = False
                                 thisDate = getDateFromRadarFolder(thisDir)
                                 pathList.append(datapath)
                                 dateList.append(thisDate)
                     dateList.sort()
                     if walk:
                         pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
                     else:
                         pattern_path = multi_path[0]
                     if path_empty:
                         raise schainpy.admin.SchainError("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
                     else:
                         if not dateList:
                             raise schainpy.admin.SchainError("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
                     if include_path:
                         return dateList, pathList
                     return dateList
                 def setup(self, **kwargs):
                     self.set_kwargs(**kwargs)
                     if not self.ext.startswith('.'):
                         self.ext = '.{}'.format(self.ext)
                     if self.server is not None:
                         if 'tcp://' in self.server:
                             address = server
                         else:
                             address = 'ipc:///tmp/%s' % self.server
                         self.server = address
                         self.context = zmq.Context()
                         self.receiver = self.context.socket(zmq.PULL)
                         self.receiver.connect(self.server)
                         time.sleep(0.5)
                         print('[Starting] ReceiverData from {}'.format(self.server))
                     else:
                         self.server = None
                         if self.path == None:
                             raise ValueError("[Reading] The path is not valid")
                         if self.online:
                             log.log("[Reading] Searching files in online mode...", self.name)
                             for nTries in range(self.nTries):
                                 fullpath = self.searchFilesOnLine(self.path, self.startDate,
                                     self.endDate, self.expLabel, self.ext, self.walk,
                                     self.filefmt, self.folderfmt)
                                 try:
                                     fullpath = next(fullpath)
                                 except:
                                     fullpath = None
                                 if fullpath:
                                     break
                                 log.warning(
                                     'Waiting {} sec for a valid file in {}: try {} ...'.format(
                                         self.delay, self.path, nTries + 1),
                                     self.name)
                                 time.sleep(self.delay)
                             if not(fullpath):
                                 raise schainpy.admin.SchainError(
                                     'There isn\'t any valid file in {}'.format(self.path))
                             pathname, filename = os.path.split(fullpath)
                             self.year = int(filename[1:5])
                             self.doy = int(filename[5:8])
                             self.set = int(filename[8:11]) - 1
                         else:
                             log.log("Searching files in {}".format(self.path), self.name)
                             self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
                                 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
                         self.setNextFile()
                     return
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
 . + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                     self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
                 def getFirstHeader(self):
                     raise NotImplementedError
                 def getData(self):
                     raise NotImplementedError
                 def hasNotDataInBuffer(self):
                     raise NotImplementedError
                 def readBlock(self):
                     raise NotImplementedError
                 def isEndProcess(self):
                     return self.flagNoMoreFiles
                 def printReadBlocks(self):
                     print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
                 def printTotalBlocks(self):
                     print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
                 def run(self, **kwargs):
                     """
                     Arguments:
                         path        :
                         startDate   :
                         endDate     :
                         startTime   :
                         endTime     :
                         set         :
                         expLabel    :
                         ext         :
                         online      :
                         delay       :
                         walk        :
                         getblock    :
                         nTxs        :
                         realtime    :
                         blocksize   :
                         blocktime   :
                         skip        :
                         cursor      :
                         warnings    :
                         server      :
                         verbose     :
                         format      :
                         oneDDict    :
                         twoDDict    :
                         independentParam    :
                     """
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     if self.server is None:
                         self.getData()
                     else:
                         self.getFromServer()
             class JRODataWriter(Reader):
                 """
                 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 setFile = None
                 profilesPerBlock = None
                 blocksPerFile = None
                 nWriteBlocks = 0
                 fileDate = None
                 def __init__(self, dataOut=None):
                     raise NotImplementedError
                 def hasAllDataInBuffer(self):
                     raise NotImplementedError
                 def setBlockDimension(self):
                     raise NotImplementedError
                 def writeBlock(self):
                     raise NotImplementedError
                 def putData(self):
                     raise NotImplementedError
                 def getDtypeWidth(self):
                     dtype_index = get_dtype_index(self.dtype)
                     dtype_width = get_dtype_width(dtype_index)
                     return dtype_width
                 def getProcessFlags(self):
                     processFlags = 0
                     dtype_index = get_dtype_index(self.dtype)
                     procflag_dtype = get_procflag_dtype(dtype_index)
                     processFlags += procflag_dtype
                     if self.dataOut.flagDecodeData:
                         processFlags += PROCFLAG.DECODE_DATA
                     if self.dataOut.flagDeflipData:
                         processFlags += PROCFLAG.DEFLIP_DATA
                     if self.dataOut.code is not None:
                         processFlags += PROCFLAG.DEFINE_PROCESS_CODE
                     if self.dataOut.nCohInt > 1:
                         processFlags += PROCFLAG.COHERENT_INTEGRATION
                     if self.dataOut.type == "Spectra":
                         if self.dataOut.nIncohInt > 1:
                             processFlags += PROCFLAG.INCOHERENT_INTEGRATION
                         if self.dataOut.data_dc is not None:
                             processFlags += PROCFLAG.SAVE_CHANNELS_DC
                         if self.dataOut.flagShiftFFT:
                             processFlags += PROCFLAG.SHIFT_FFT_DATA
                     return processFlags
                 def setBasicHeader(self):
                     self.basicHeaderObj.size = self.basicHeaderSize  # bytes
                     self.basicHeaderObj.version = self.versionFile
                     self.basicHeaderObj.dataBlock = self.nTotalBlocks
                     utc = numpy.floor(self.dataOut.utctime)
                     milisecond = (self.dataOut.utctime - utc) * 1000.0
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = milisecond
                     self.basicHeaderObj.timeZone = self.dataOut.timeZone
                     self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
                     self.basicHeaderObj.errorCount = self.dataOut.errorCount
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj self.
                     Return:
                         None
                     """
                     raise NotImplementedError
                 def __writeFirstHeader(self):
                     """
                     Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
                     Affected:
                         __dataType
                     Return:
                         None
                     """
             #        CALCULAR PARAMETROS
                     sizeLongHeader = self.systemHeaderObj.size + \
                         self.radarControllerHeaderObj.size + self.processingHeaderObj.size
                     self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
                     self.basicHeaderObj.write(self.fp)
                     self.systemHeaderObj.write(self.fp)
                     self.radarControllerHeaderObj.write(self.fp)
                     self.processingHeaderObj.write(self.fp)
                 def __setNewBlock(self):
                     """
                     Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
                     Return:
 :    si no pudo escribir nada
 :    Si escribio el Basic el First Header
                     """
                     if self.fp == None:
                         self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
                         self.basicHeaderObj.write(self.fp)
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     """
                     Selecciona el bloque siguiente de datos y los escribe en un file
                     Return:
 :    Si no hizo pudo escribir el bloque de datos
 :    Si no pudo escribir el bloque de datos
                     """
                     if not(self.__setNewBlock()):
                         return 0
                     self.writeBlock()
                     print("[Writing] Block No. %d/%d" % (self.blockIndex,
                                                          self.processingHeaderObj.dataBlocksPerFile))
                     return 1
                 def setNextFile(self):
                     """Determina el siguiente file que sera escrito
                     Affected:
                         self.filename
                         self.subfolder
                         self.fp
                         self.setFile
                         self.flagIsNewFile
                     Return:
 :    Si el archivo no puede ser escrito
 :    Si el archivo esta listo para ser escrito
                     """
                     ext = self.ext
                     path = self.path
                     if self.fp != None:
                         self.fp.close()
                     if not os.path.exists(path):
                         os.mkdir(path)
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
                     fullpath = os.path.join(path, subfolder)
                     setFile = self.setFile
                     if not(os.path.exists(fullpath)):
                         os.mkdir(fullpath)
                         setFile = -1  # inicializo mi contador de seteo
                     else:
                         filesList = os.listdir(fullpath)
                         if len(filesList) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 # inicializo mi contador de seteo al seteo del ultimo file
                                 setFile = int(filen[8:11])
                             else:
                                 setFile = -1
                         else:
                             setFile = -1  # inicializo mi contador de seteo
                     setFile += 1
                     # If this is a new day it resets some values
                     if self.dataOut.datatime.date() > self.fileDate:
                         setFile = 0
                         self.nTotalBlocks = 0
                     filen = '{}{:04d}{:03d}{:03d}{}'.format(
                             self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
                     filename = os.path.join(path, subfolder, filen)
                     fp = open(filename, 'wb')
                     self.blockIndex = 0
                     self.filename = filename
                     self.subfolder = subfolder
                     self.fp = fp
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     self.fileDate = self.dataOut.datatime.date()
                     self.setFirstHeader()
                     print('[Writing] Opening file: %s' % self.filename)
                     self.__writeFirstHeader()
                     return 1
                 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
                     """
                     Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
                     Inputs:
                         path                :    directory where data will be saved
                         profilesPerBlock    :    number of profiles per block
                         set                 :    initial file set
                         datatype            :    An integer number that defines data type:
 : int8  (1 byte)
 : int16 (2 bytes)
 : int32 (4 bytes)
 : int64 (8 bytes)
 : float32 (4 bytes)
 : double64 (8 bytes)
                     Return:
 :    Si no realizo un buen seteo
 :    Si realizo un buen seteo
                     """
                     if ext == None:
                         ext = self.ext
                     self.ext = ext.lower()
                     self.path = path
                     if set is None:
                         self.setFile = -1
                     else:
                         self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile
                     self.profilesPerBlock = profilesPerBlock
                     self.dataOut = dataOut
                     self.fileDate = self.dataOut.datatime.date()
                     self.dtype = self.dataOut.dtype
                     if datatype is not None:
                         self.dtype = get_numpy_dtype(datatype)
                     if not(self.setNextFile()):
                         print("[Writing] There isn't a next file")
                         return 0
                     self.setBlockDimension()
                     return 1
                 def run(self, dataOut, path, blocksPerFile=100, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
                                    set=set, ext=ext, datatype=datatype, **kwargs)
                         self.isConfig = True
                     self.dataOut = dataOut
                     self.putData()
                     return self.dataOut
             @MPDecorator
             class printInfo(Operation):
                 def __init__(self):
                     Operation.__init__(self)
                     self.__printInfo = True
                 def run(self, dataOut, headers = ['systemHeaderObj', 'radarControllerHeaderObj', 'processingHeaderObj']):
                     if self.__printInfo == False:
                         return
                     for header in headers:
                         if hasattr(dataOut, header):
                             obj = getattr(dataOut, header)
                             if hasattr(obj, 'printInfo'):
                                 obj.printInfo()
                             else:
                                 print(obj)
                         else:
                             log.warning('Header {} Not found in object'.format(header))
                     self.__printInfo = False

schainpy/model/io/jroIO_param.py +18 -7

             from email.utils import localtime
             import os
             import time
             import datetime
             import numpy
             import h5py
             import schainpy.admin
             from schainpy.model.data.jrodata import *
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.io.jroIO_base import *
             from schainpy.utils import log
             class HDFReader(Reader, ProcessingUnit):
                 """Processing unit to read HDF5 format files
                 This unit reads HDF5 files created with `HDFWriter` operation contains
                 by default two groups Data and Metadata all variables would be saved as `dataOut`
                 attributes.
                 It is possible to read any HDF5 file by given the structure in the `description`
                 parameter, also you can add extra values to metadata with the parameter `extras`.
                 Parameters:
                 -----------
                 path : str
                     Path where files are located.
                 startDate : date
                     Start date of the files
                 endDate : list
                     End date of the files
                 startTime : time
                     Start time of the files
                 endTime : time
                     End time of the files
                 description : dict, optional
                     Dictionary with the description of the HDF5 file
                 extras : dict, optional
                     Dictionary with extra metadata to be be added to `dataOut`
                 Examples
                 --------
                 desc = {
                     'Data': {
                         'data_output': ['u', 'v', 'w'],
                         'utctime': 'timestamps',
                     }  ,
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 desc = {
                     'Data': {
                         'data_output': 'winds',
                         'utctime': 'timestamps'
                     },
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 extras = {
                     'timeZone': 300
                 }
                 reader = project.addReadUnit(
                     name='HDFReader',
                     path='/path/to/files',
                     startDate='2019/01/01',
                     endDate='2019/01/31',
                     startTime='00:00:00',
                     endTime='23:59:59',
                     # description=json.dumps(desc),
                     # extras=json.dumps(extras),
                     )
                 """
                 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                     self.ext = ".hdf5"
                     self.optchar = "D"
                     self.meta = {}
                     self.data = {}
                     self.open_file = h5py.File
                     self.open_mode = 'r'
                     self.description = {}
                     self.extras = {}
                     self.filefmt = "*%Y%j***"
                     self.folderfmt = "*%Y%j"
                     self.utcoffset = 0
+                    self.filter  = None
+                    self.dparam  = None
                 def setup(self, **kwargs):
                     self.set_kwargs(**kwargs)
                     if not self.ext.startswith('.'):
                         self.ext = '.{}'.format(self.ext)
                     if self.online:
                         log.log("Searching files in online mode...", self.name)
                         for nTries in range(self.nTries):
                             fullpath = self.searchFilesOnLine(self.path, self.startDate,
                                 self.endDate, self.expLabel, self.ext, self.walk,
-                                self.filefmt, self.folderfmt)
+                                self.filefmt, self.folderfmt,self.filter)
                             try:
                                 fullpath = next(fullpath)
                             except:
                                 fullpath = None
                             if fullpath:
                                 break
                             log.warning(
                                 'Waiting {} sec for a valid file in {}: try {} ...'.format(
                                     self.delay, self.path, nTries + 1),
                                 self.name)
                             time.sleep(self.delay)
                         if not(fullpath):
                             raise schainpy.admin.SchainError(
                                 'There isn\'t any valid file in {}'.format(self.path))
                         pathname, filename = os.path.split(fullpath)
                         self.year = int(filename[1:5])
                         self.doy = int(filename[5:8])
                         self.set = int(filename[8:11]) - 1
                     else:
                         log.log("Searching files in {}".format(self.path), self.name)
                         self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
-                            self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
+                            self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt,self.filter)
                     self.setNextFile()
                     return
                 def readFirstHeader(self):
                     '''Read metadata and data'''
                     self.__readMetadata()
                     self.__readData()
                     self.__setBlockList()
                     if 'type' in self.meta:
                         self.dataOut = eval(self.meta['type'])()
+                    if self.dparam:
+                        setattr(self.dataOut, "dparam", 1)
                     for attr in self.meta:
                         setattr(self.dataOut, attr, self.meta[attr])
                     self.blockIndex = 0
                     return
                 def __setBlockList(self):
                     '''
                     Selects the data within the times defined
                     self.fp
                     self.startTime
                     self.endTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     startTime = self.startTime
                     endTime = self.endTime
                     thisUtcTime = self.data['utctime'] + self.utcoffset
-                    self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
+                    try:
+                        self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
+                    except:
+                        self.interval = 0
                     thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     return
                 def __readMetadata(self):
                     '''
                     Reads Metadata
                     '''
                     meta = {}
                     if self.description:
                         for key, value in self.description['Metadata'].items():
                             meta[key] = self.fp[value][()]
                     else:
                         grp = self.fp['Metadata']
                         for name in grp:
                             meta[name] = grp[name][()]
                     if self.extras:
                         for key, value in self.extras.items():
                             meta[key] = value
                     self.meta = meta
                     return
                 def __readData(self):
                     data = {}
                     if self.description:
                         for key, value in self.description['Data'].items():
                             if isinstance(value, str):
                                 if isinstance(self.fp[value], h5py.Dataset):
                                     data[key] = self.fp[value][()]
                                 elif isinstance(self.fp[value], h5py.Group):
                                     array = []
                                     for ch in self.fp[value]:
                                         array.append(self.fp[value][ch][()])
                                     data[key] = numpy.array(array)
                             elif isinstance(value, list):
                                 array = []
                                 for ch in value:
                                     array.append(self.fp[ch][()])
                                 data[key] = numpy.array(array)
                     else:
                         grp = self.fp['Data']
                         for name in grp:
                             if isinstance(grp[name], h5py.Dataset):
                                 array = grp[name][()]
                             elif isinstance(grp[name], h5py.Group):
                                 array = []
                                 for ch in grp[name]:
                                     array.append(grp[name][ch][()])
                                 array = numpy.array(array)
                             else:
                                 log.warning('Unknown type: {}'.format(name))
                             if name in self.description:
                                 key = self.description[name]
                             else:
                                 key = name
                             data[key] = array
                     self.data = data
                     return
                 def getData(self):
                     for attr in self.data:
                         if self.data[attr].ndim == 1:
                             setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
                         else:
-                            setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
+                            if self.dparam:
+                                setattr(self.dataOut, attr, self.data[attr])
+                            else:
+                                setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
                     self.dataOut.flagNoData = False
                     self.blockIndex += 1
                     log.log("Block No. {}/{} -> {}".format(
                         self.blockIndex,
                         self.blocksPerFile,
                         self.dataOut.datatime.ctime()), self.name)
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     if self.blockIndex == self.blocksPerFile:
                         self.setNextFile()
                     self.getData()
                     return
             @MPDecorator
             class HDFWriter(Operation):
                 """Operation to write HDF5 files.
                 The HDF5 file contains by default two groups Data and Metadata where
                 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
                 parameters, data attributes are normaly time dependent where the metadata
                 are not.
                 It is possible to customize the structure of the HDF5 file with the
                 optional description parameter see the examples.
                 Parameters:
                 -----------
                 path : str
                     Path where files will be saved.
                 blocksPerFile : int
                     Number of blocks per file
                 metadataList : list
                     List of the dataOut attributes that will be saved as metadata
                 dataList : int
                     List of the dataOut attributes that will be saved as data
                 setType : bool
                     If True the name of the files corresponds to the timestamp of the data
                 description : dict, optional
                     Dictionary with the desired description of the HDF5 file
                 Examples
                 --------
                 desc = {
                     'data_output': {'winds': ['z', 'w', 'v']},
                     'utctime': 'timestamps',
                     'heightList': 'heights'
                 }
                 desc = {
                     'data_output': ['z', 'w', 'v'],
                     'utctime': 'timestamps',
                     'heightList': 'heights'
                 }
                 desc = {
                     'Data': {
                         'data_output': 'winds',
                         'utctime': 'timestamps'
                     },
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 writer = proc_unit.addOperation(name='HDFWriter')
                 writer.addParameter(name='path', value='/path/to/file')
                 writer.addParameter(name='blocksPerFile', value='32')
                 writer.addParameter(name='metadataList', value='heightList,timeZone')
                 writer.addParameter(name='dataList',value='data_output,utctime')
                 # writer.addParameter(name='description',value=json.dumps(desc))
                 """
                 ext = ".hdf5"
                 optchar = "D"
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 currentDay = None
                 lastTime = None
                 last_Azipos = None
                 last_Elepos = None
                 mode       = None
                 #-----------------------
                 Typename = None
                 mask = False
                 def __init__(self):
                     Operation.__init__(self)
                     return
                 def set_kwargs(self, **kwargs):
                     for key, value in kwargs.items():
                         setattr(self, key, value)
                 def set_kwargs_obj(self,obj, **kwargs):
                     for key, value in kwargs.items():
                         setattr(obj, key, value)
                 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None,type_data=None, localtime=True, **kwargs):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataList = [s.strip() for s in dataList]
                     self.setType = setType
                     if self.setType == "weather":
                         self.set_kwargs(**kwargs)
                         self.set_kwargs_obj(self.dataOut,**kwargs)
                         self.weather_vars = {
                             'S' : 0,
                             'V' : 1,
                             'W' : 2,
                             'SNR' : 3,
                             'Z' : 4,
                             'D' : 5,
                             'P' : 6,
                             'R' : 7,
                         }
                     if localtime:
                         self.getDateTime = datetime.datetime.fromtimestamp
                     else:
                         self.getDateTime = datetime.datetime.utcfromtimestamp
                     self.description = description
                     self.type_data=type_data
                     if self.metadataList is None:
                         self.metadataList = self.dataOut.metadata_list
                     dsList = []
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         if hasattr(self.dataOut, self.dataList[i]):
                             dataAux = getattr(self.dataOut, self.dataList[i])
                             if self.setType == 'weather' and self.dataList[i] == 'data_param':
                                 dataAux = dataAux[:,self.weather_vars[self.weather_var],:]
                             dsDict['variable'] = self.dataList[i]
                         else:
                             log.warning('Attribute {} not found in dataOut'.format(self.dataList[i]), self.name)
                             continue
                         if dataAux is None:
                             continue
                         elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
                             dsDict['nDim'] = 0
                         else:
                             dsDict['nDim'] = len(dataAux.shape)
                             dsDict['shape'] = dataAux.shape
                             dsDict['dsNumber'] = dataAux.shape[0]
                             dsDict['dtype'] = dataAux.dtype
                         dsList.append(dsDict)
                     self.dsList = dsList
                     self.currentDay = self.dataOut.datatime.date()
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     dt = self.getDateTime(currentTime)
                     dataDay = int(dt.strftime('%j'))
                     if self.lastTime is None:
                         self.lastTime = currentTime
                         self.currentDay = dataDay
                         return False
                     timeDiff = currentTime - self.lastTime
                     #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                     if dataDay != self.currentDay:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > 3*60*60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
                         dataList=[], setType=None, description={}, mode= None,
                         type_data=None, Reset = False, localtime=True, **kwargs):
                     if Reset:
                         self.isConfig = False
                         self.closeFile()
                         self.lastTime = None
                         self.blockIndex = 0
                     self.dataOut = dataOut
                     self.mode    = mode
                     if not(self.isConfig):
                         self.setup(path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList,
                                    setType=setType, description=description,type_data=type_data,
                                    localtime=localtime, **kwargs)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     dt = self.getDateTime(self.dataOut.utctime)
                     if self.setType == 'weather':
                         subfolder = dt.strftime('%Y-%m-%dT%H-00-00')
                     else:
                         subfolder = dt.strftime('d%Y%j')
                     fullpath = os.path.join(path, subfolder)
                     if os.path.exists(fullpath):
                         filesList = os.listdir(fullpath)
                         filesList = [k for k in filesList if k.startswith(self.optchar)]
                         if len( filesList ) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        dt.year,
                                                        int(dt.strftime('%j')),
                                                        setFile,
                                                        ext )
                     elif self.setType == "weather":
                         #SOPHY_20200505_140215_E10.0_Z.h5
                         #SOPHY_20200505_140215_A40.0_Z.h5
                         if self.dataOut.flagMode == 1: #'AZI' #PPI
                             ang_type = 'E'
                             len_aux = int(self.dataOut.data_ele.shape[0]/4)
                             mean = numpy.mean(self.dataOut.data_ele[len_aux:-len_aux])
                             ang_    = round(mean,1)
                         elif self.dataOut.flagMode == 0: #'ELE' #RHI
                             ang_type = 'A'
                             len_aux = int(self.dataOut.data_azi.shape[0]/4)
                             mean = numpy.mean(self.dataOut.data_azi[len_aux:-len_aux])
                             ang_    = round(mean,1)
                         file = '%s_%2.2d%2.2d%2.2d_%2.2d%2.2d%2.2d_%s%2.1f_%s%s' % (
                             'SOPHY',
                                                        dt.year,
                                                        dt.month,
                                                        dt.day,
                                                        dt.hour,
                                                        dt.minute,
                                                        dt.second,
                                                        ang_type,
                                                        ang_,
                                                        self.weather_var,
                                                        ext )
                     else:
                         setFile = dt.hour*60+dt.minute
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        dt.year,
                                                        int(dt.strftime('%j')),
                                                        setFile,
                                                        ext )
                     self.filename = os.path.join( path, subfolder, file )
                     self.fp = h5py.File(self.filename, 'w')
                     #write metadata
                     self.writeMetadata(self.fp)
                     #Write data
                     self.writeData(self.fp)
                 def getLabel(self, name, x=None):
                     if x is None:
                         if 'Data' in self.description:
                             data = self.description['Data']
                             if 'Metadata' in self.description:
                                 data.update(self.description['Metadata'])
                         else:
                             data = self.description
                         if name in data:
                             if isinstance(data[name], str):
                                 return data[name]
                             elif isinstance(data[name], list):
                                 return None
                             elif isinstance(data[name], dict):
                                 for key, value in data[name].items():
                                     return key
                         return name
                     else:
                         if 'Data' in self.description:
                             data = self.description['Data']
                             if 'Metadata' in self.description:
                                 data.update(self.description['Metadata'])
                         else:
                             data = self.description
                         if name in data:
                             if isinstance(data[name], list):
                                 return data[name][x]
                             elif isinstance(data[name], dict):
                                 for key, value in data[name].items():
                                     return value[x]
                         if 'cspc' in name:
                             return 'pair{:02d}'.format(x)
                         else:
                             return 'channel{:02d}'.format(x)
                 def writeMetadata(self, fp):
                     if self.description:
                         if 'Metadata' in self.description:
                             grp = fp.create_group('Metadata')
                         else:
                             grp = fp
                     else:
                         grp = fp.create_group('Metadata')
                     for i in range(len(self.metadataList)):
                         if not hasattr(self.dataOut, self.metadataList[i]):
                             log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
                             continue
                         value = getattr(self.dataOut, self.metadataList[i])
                         if isinstance(value, bool):
                             if value is True:
                                 value = 1
                             else:
                                 value = 0
                         grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
                     return
                 def writeData(self, fp):
                     if self.description:
                         if 'Data' in self.description:
                             grp = fp.create_group('Data')
                         else:
                             grp = fp
                     else:
                         grp = fp.create_group('Data')
                     dtsets = []
                     data = []
                     for dsInfo in self.dsList:
                         if dsInfo['nDim'] == 0:
                             ds = grp.create_dataset(
                                 self.getLabel(dsInfo['variable']),
                                 (self.blocksPerFile, ),
                                 chunks=True,
                                 dtype=numpy.float64)
                             dtsets.append(ds)
                             data.append((dsInfo['variable'], -1))
                         else:
                             label = self.getLabel(dsInfo['variable'])
                             if label is not None:
                                 sgrp = grp.create_group(label)
                             else:
                                 sgrp = grp
                             if self.blocksPerFile == 1:
                                 shape = dsInfo['shape'][1:]
                             else:
                                 shape = (self.blocksPerFile, ) + dsInfo['shape'][1:]
                             for i in range(dsInfo['dsNumber']):
                                 ds = sgrp.create_dataset(
                                     self.getLabel(dsInfo['variable'], i),
                                     shape,
                                     chunks=True,
                                     dtype=dsInfo['dtype'],
                                     compression='gzip',
                                     )
                                 dtsets.append(ds)
                                 data.append((dsInfo['variable'], i))
                     fp.flush()
                     log.log('Creating file: {}'.format(fp.filename), self.name)
                     self.ds = dtsets
                     self.data = data
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
                         self.closeFile()
                         self.setNextFile()
                     for i, ds in enumerate(self.ds):
                         attr, ch = self.data[i]
                         if ch == -1:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)
                         else:
                             if self.blocksPerFile == 1:
                                 mask = self.dataOut.data_param[:,3,:][ch] < self.mask
                                 tmp = getattr(self.dataOut, attr)[:,self.weather_vars[self.weather_var],:][ch]
                                 if self.mask:
                                     tmp[mask] = numpy.nan
                                 ds[:] = tmp
                             else:
                                 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
                     self.fp.flush()
                     self.blockIndex += 1
                     log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
                     return
                 def closeFile(self):
                     if self.blockIndex != self.blocksPerFile:
                         for ds in self.ds:
                             ds.resize(self.blockIndex, axis=0)
                     if self.fp:
                         self.fp.flush()
                         self.fp.close()
                 def close(self):
                     self.closeFile()

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages