##// END OF EJS Templates
schain
RSS
last 30-16:19
avaldezp -
r1405:49f004b59e80
parent child
Show More
Side by Side Unified
Context file:
r1405:49f004b59e80 Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -0,0 +1,41
1 # Ing. AVP
2 # 01/11/2021
3 # ARCHIVO DE LECTURA
4 import os, sys
5 import datetime
6 import time
7 from schainpy.controller import Project
8 print("----[Setup]-RadarMeteorologico--------")
9 Vel = 6
10 modo_proc = 1
11 #-----------PATH DE DATOS-----------------------#
12 path = "/DATA_RM/10"
13 path_ped = "/DATA_RM/TEST_PEDESTAL/P20211111-173856"
14 print("----[OPCIONES]------------------------")
15 op_plot = 0
16 op_integration = 0
17 op_save = 0
18 op_plot_spec = 0
19
20 ########################SIGNAL CHAIN ##################################
21 desc = "USRP_test"
22 filename = "USRP_processing.xml"
23 controllerObj = Project()
24 controllerObj.setup(id = '191', name='Test_USRP', description=desc)
25 ######################## UNIDAD DE LECTURA#############################
26 readUnitConfObj = controllerObj.addReadUnit(datatype='DigitalRFReader',
27 path=path,
28 startDate="2021/11/11",#today,
29 endDate="2021/12/30",#today,
30 startTime='17:39:25',
31 endTime='23:59:59',
32 delay=0,
33 #set=0,
34 online=0,
35 walk=1,
36 ippKm = 60)
37
38 opObj11 = readUnitConfObj.addOperation(name='printInfo')
39
40 procUnitConfObjA = controllerObj.addProcUnit(datatype='VoltageProc',inputId=readUnitConfObj.getId())
41 controllerObj.start()
Show file before | Show file after | Hide comments
@@ -0,0 +1,72
1 import os, sys
2 import datetime
3 import time
4 from schainpy.controller import Project
5
6 desc = "USRP_test"
7 filename = "USRP_processing.xml"
8 controllerObj = Project()
9 controllerObj.setup(id = '191', name='Test_USRP', description=desc)
10
11 ############## USED TO PLOT IQ VOLTAGE, POWER AND SPECTRA #############
12 ######PATH DE LECTURA, ESCRITURA, GRAFICOS Y ENVIO WEB#################
13 path = '/home/alex/Downloads/test_rawdata'
14 path = '/DATA_RM/WR_POT_09_2'
15 figpath = '/home/alex/Downloads'
16 figpath_pp = "/home/soporte/Pictures/TEST_POT"
17 ################# RANGO DE PLOTEO######################################
18 dBmin = '30'
19 dBmax = '60'
20 xmin = '0'
21 xmax ='24'
22 ymin = '0'
23 ymax = '600'
24 ########################FECHA##########################################
25 str = datetime.date.today()
26 today = str.strftime("%Y/%m/%d")
27 str2 = str - datetime.timedelta(days=1)
28 yesterday = str2.strftime("%Y/%m/%d")
29 ######################## UNIDAD DE LECTURA#############################
30 readUnitConfObj = controllerObj.addReadUnit(datatype='DigitalRFReader',
31 path=path,
32 startDate="2021/01/01", #"2020/01/01",#today,
33 endDate= "2021/12/01", #"2020/12/30",#today,
34 startTime='00:00:00',
35 endTime='23:59:59',
36 delay=0,
37 #set=0,
38 online=0,
39 walk =1,
40 ippKm = 60 )
41
42 opObj11 = readUnitConfObj.addOperation(name='printInfo')
43 #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
44 #######################################################################
45 ################ OPERACIONES DOMINIO DEL TIEMPO########################
46 #######################################################################
47
48 procUnitConfObjA = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
49 '''
50 opObj11 = procUnitConfObjA.addOperation(name='PulsePairVoltage', optype='other')
51 opObj11.addParameter(name='n', value='256', format='int')
52 opObj11.addParameter(name='removeDC', value=1, format='int')
53 '''
54
55 _type="iq"
56 opObj10 = procUnitConfObjA.addOperation(name='ScopePlot', optype='external')
57 #opObj10.addParameter(name='id', value='12')
58 opObj10.addParameter(name='wintitle', value=_type )
59 opObj10.addParameter(name='type', value=_type)
60
61
62 '''
63 type="WeatherPower"
64 opObj10 = procUnitConfObjA.addOperation(name='PulsepairPowerPlot', optype='external')
65 #opObj10.addParameter(name='id', value='12')
66 opObj10.addParameter(name='wintitle', value=type )
67
68 opObj11 = procUnitConfObjA.addOperation(name='PulsepairVelocityPlot', optype='other')
69 opObj11.addParameter(name='xmax', value=8)
70 '''
71
72 controllerObj.start()
Show file before | Show file after | Hide comments
@@ -0,0 +1,118
1 # Ing. AVP
2 # 01/11/2021
3 # ARCHIVO DE LECTURA
4 import os, sys
5 import datetime
6 import time
7 from schainpy.controller import Project
8 print("----[Setup]-RadarMeteorologico--------")
9 Vel = 10
10 modo_proc = 0 # 0 Pulse Pair 1 Spectros
11 #-----------PATH DE DATOS-----------------------#
12 #------------VERIFICAR SIEMPRE LA FECHA DE LA DATA
13 path = "/DATA_RM/10"
14 path_ped = "/DATA_RM/TEST_PEDESTAL/P20211110-171003"
15 figpath_pp = "/home/soporte/Pictures/TEST_POT"
16 figpath_ppi_pp = "/home/soporte/Pictures/ppi_PP_30DIC_4"
17 #-------------------------------------------------------------------
18 print("----[OPCIONES]------------------------")
19 op_plot = 0
20 op_integration = 1
21 op_save = 1
22 op_plot_spec = 0
23 #-------------------------------------
24 ################# RANGO DE PLOTEO######################################
25 dBmin = '1'
26 dBmax = '85'
27 xmin = '17.1'
28 xmax = '17.25'
29 ymin = '0'
30 ymax = '600'
31 #-------------------NRO Perfiles PROCESAMIENTO --------------------
32 V=Vel
33 IPP=400*1e-6
34 n= int(1/(V*IPP))
35 print("* n - NRO Perfiles Proc:", n )
36 time.sleep(3)
37
38 #------------------------SIGNAL CHAIN ------------------------------
39 desc = "USRP_test"
40 filename = "USRP_processing.xml"
41 controllerObj = Project()
42 controllerObj.setup(id = '191', name='Test_USRP', description=desc)
43 ######################## UNIDAD DE LECTURA#############################
44 readUnitConfObj = controllerObj.addReadUnit(datatype = 'DigitalRFReader',
45 path = path,
46 startDate= "2021/11/10",#today,
47 endDate = "2021/12/30",#today,
48 startTime= '00:00:25',
49 endTime = '23:59:59',
50 delay = 0,
51 online = 0,
52 walk = 1,
53 ippKm = 60)
54
55 opObj11 = readUnitConfObj.addOperation(name='printInfo')
56 procUnitConfObjA = controllerObj.addProcUnit(datatype='VoltageProc',inputId=readUnitConfObj.getId())
57
58 opObj11 = procUnitConfObjA.addOperation(name='selectHeights')
59 opObj11.addParameter(name='minIndex', value='1', format='int')
60 # opObj11.addParameter(name='maxIndex', value='10000', format='int')
61 opObj11.addParameter(name='maxIndex', value='400', format='int')
62
63 if modo_proc ==0:
64 #----------------------------------------PULSE PAIR --------------------------------------------------#
65 opObj11 = procUnitConfObjA.addOperation(name='PulsePair', optype='other')
66 opObj11.addParameter(name='n', value=int(n), format='int')#10 VOY A USAR 250 DADO QUE LA VELOCIDAD ES 10 GRADOS
67 #opObj11.addParameter(name='removeDC', value=1, format='int')
68 #------------------------ METODO Parametros -----------------------------------------------------------
69 procUnitConfObjB= controllerObj.addProcUnit(datatype='ParametersProc',inputId=procUnitConfObjA.getId())
70 if op_plot==1:
71 opObj11 = procUnitConfObjB.addOperation(name='GenericRTIPlot',optype='external')
72 opObj11.addParameter(name='attr_data', value='dataPP_POW')
73 opObj11.addParameter(name='colormap', value='jet')
74 #opObj11.addParameter(name='xmin', value=xmin)
75 #opObj11.addParameter(name='xmax', value=xmax)
76 opObj11.addParameter(name='zmin', value=dBmin)
77 opObj11.addParameter(name='zmax', value=dBmax)
78 opObj11.addParameter(name='save', value=figpath_pp)
79 opObj11.addParameter(name='showprofile', value=0)
80 opObj11.addParameter(name='save_period', value=50)
81
82 ####################### METODO ESCRITURA #######################################################################
83
84 if op_integration==1:
85 V=V
86 blocksPerfile=100
87 print("* Velocidad del Pedestal:",V)
88 tmp_blocksPerfile = 100
89 f_a_p= int(tmp_blocksPerfile/V)
90
91 opObj11 = procUnitConfObjB.addOperation(name='PedestalInformation')
92 opObj11.addParameter(name='path_ped', value=path_ped)
93 #opObj11.addParameter(name='path_adq', value=path_adq)
94 opObj11.addParameter(name='t_Interval_p', value='0.01', format='float')
95 opObj11.addParameter(name='blocksPerfile', value=blocksPerfile, format='int')
96 opObj11.addParameter(name='n_Muestras_p', value='100', format='float')
97 opObj11.addParameter(name='f_a_p', value=f_a_p, format='int')
98 opObj11.addParameter(name='online', value='0', format='int')
99
100 opObj11 = procUnitConfObjB.addOperation(name='Block360')
101 opObj11.addParameter(name='n', value='10', format='int')
102 opObj11.addParameter(name='mode', value=modo_proc, format='int')
103
104 # este bloque funciona bien con divisores de 360 no olvidar 0 10 20 30 40 60 90 120 180
105
106 opObj11= procUnitConfObjB.addOperation(name='WeatherPlot',optype='other')
107 opObj11.addParameter(name='save', value=figpath_ppi_pp)
108 opObj11.addParameter(name='save_period', value=1)
109
110 if op_save==1:
111 opObj10 = procUnitConfObjB.addOperation(name='HDFWriter')
112 opObj10.addParameter(name='path',value=figpath_ppi_pp)
113 #opObj10.addParameter(name='mode',value=0)
114 opObj10.addParameter(name='blocksPerFile',value='100',format='int')
115 opObj10.addParameter(name='metadataList',value='utctimeInit,timeZone,paramInterval,profileIndex,channelList,heightList,flagDataAsBlock',format='list')
116 opObj10.addParameter(name='dataList',value='dataPP_POW,dataPP_DOP,azimuth,utctime',format='list')#,format='list'
117
118 controllerObj.start()
Show file before | Show file after | Hide comments
@@ -0,0 +1,14
1 import numpy
2 a= numpy.array([0,1,2,3,4,5,10,11,12,18,19,20,21,22,23,24,25,26,27,28])
3 print(a)
4 list=[]
5 list2=[]
6 for i in reversed(range(1,len(a))):
7 dif=int(a[i])-int(a[i-1])
8 print(i,a[i],dif )
9 if dif>1:
10 list.append(i-1)
11 list2.append(dif-1)
12 print("result")
13 print(list)
14 print(list2)
Show file before | Show file after | Hide comments
@@ -0,0 +1,147
1 #!python
2 '''
3 '''
4
5 import os, sys
6 import datetime
7 import time
8
9 #path = os.path.dirname(os.getcwd())
10 #path = os.path.dirname(path)
11 #sys.path.insert(0, path)
12
13 from schainpy.controller import Project
14
15 desc = "USRP_test"
16 filename = "USRP_processing.xml"
17 controllerObj = Project()
18 controllerObj.setup(id = '191', name='Test_USRP', description=desc)
19
20 ############## USED TO PLOT IQ VOLTAGE, POWER AND SPECTRA #############
21
22 #######################################################################
23 ######PATH DE LECTURA, ESCRITURA, GRAFICOS Y ENVIO WEB#################
24 #######################################################################
25 #path = '/media/data/data/vientos/57.2063km/echoes/NCO_Woodman'
26 #path = '/DATA_RM/TEST_INTEGRACION'
27 #path = '/DATA_RM/TEST_ONLINE'
28 #path = '/DATA_RM/TEST_INTEGRACION/ADQ_OFFLINE/'
29 # ULTIMO TEST 22 DE SEPTIEMBRE
30 path = '/DATA_RM/USRP_22'
31 #path_pp = '/DATA_RM/TEST_HDF5'
32 # UTIMO TEST 22 DE SEPTIEMBRE
33 path_pp = '/DATA_RM/TEST_HDF5_PP_22'
34 ######################################################
35 ##### OJO TENER EN CUENTA EL n= para el Pulse Pair ###
36 ######################################################
37 ######## BUSCAMOS EL numero de IPP equivalente 1Β°#####
38 ######## Sea V la velocidad del Pedestal en Β°/seg#####
39 ######## 1Β° sera Recorrido en un tiempo de 1/V ######
40 ######## IPP del Radar 400 useg --> 60 Km ############
41 ######## n = 1/(V*IPP) , NUMERO DE IPP #############
42 ######## n = 1/(V*IPP) #############################
43 V=2
44 IPP=400*1e-6
45 n= 1/(V*IPP)
46 print("n numero de Perfiles a procesar con Pulse Pair: ", n)
47
48
49
50
51 figpath = '/home/soporte/Pictures/TEST_INTEGRACION_IMG'
52 #remotefolder = "/home/wmaster/graficos"
53 #######################################################################
54 ################# RANGO DE PLOTEO######################################
55 #######################################################################
56 dBmin = '-5'
57 dBmax = '20'
58 xmin = '0'
59 xmax ='24'
60 ymin = '0'
61 ymax = '600'
62 #######################################################################
63 ########################FECHA##########################################
64 #######################################################################
65 str = datetime.date.today()
66 today = str.strftime("%Y/%m/%d")
67 str2 = str - datetime.timedelta(days=1)
68 yesterday = str2.strftime("%Y/%m/%d")
69 #######################################################################
70 ######################## UNIDAD DE LECTURA#############################
71 #######################################################################
72 readUnitConfObj = controllerObj.addReadUnit(datatype='DigitalRFReader',
73 path=path,
74 startDate="2021/01/01",#today,
75 endDate="2021/12/30",#today,
76 startTime='00:00:00',
77 endTime='23:59:59',
78 delay=0,
79 #set=0,
80 online=0,
81 walk=1,
82 ippKm = 60)
83
84 opObj11 = readUnitConfObj.addOperation(name='printInfo')
85 #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
86 #######################################################################
87 ################ OPERACIONES DOMINIO DEL TIEMPO########################
88 #######################################################################
89
90 procUnitConfObjA = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
91
92 #
93 # codigo64='1,1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,1,1,1,0,1,1,0,1,0,0,0,1,1,1,0,1,1,1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,1,0,1,1,1,0,0,0,1,0,'+\
94 # '1,1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,1,1,1,0,1,1,0,1,0,0,0,1,1,1,0,1,0,0,0,1,0,0,1,0,0,0,0,1,1,1,0,1,1,1,1,0,1,1,0,1,0,0,0,1,1,1,0,1'
95
96 #opObj11 = procUnitConfObjA.addOperation(name='setRadarFrequency')
97 #opObj11.addParameter(name='frequency', value='70312500')
98 opObj11 = procUnitConfObjA.addOperation(name='PulsePair', optype='other')
99 opObj11.addParameter(name='n', value=int(n), format='int')#10 VOY A USAR 250 DADO QUE LA VELOCIDAD ES 10 GRADOS
100 opObj11.addParameter(name='removeDC', value=1, format='int')
101 # Ploteo TEST
102 '''
103 opObj11 = procUnitConfObjA.addOperation(name='PulsepairPowerPlot', optype='other')
104 opObj11 = procUnitConfObjA.addOperation(name='PulsepairSignalPlot', optype='other')
105 opObj11 = procUnitConfObjA.addOperation(name='PulsepairVelocityPlot', optype='other')
106 #opObj11.addParameter(name='xmax', value=8)
107 opObj11 = procUnitConfObjA.addOperation(name='PulsepairSpecwidthPlot', optype='other')
108 '''
109 # OJO SCOPE
110 #opObj10 = procUnitConfObjA.addOperation(name='ScopePlot', optype='external')
111 #opObj10.addParameter(name='id', value='10', format='int')
112 ##opObj10.addParameter(name='xmin', value='0', format='int')
113 ##opObj10.addParameter(name='xmax', value='50', format='int')
114 #opObj10.addParameter(name='type', value='iq')
115 ##opObj10.addParameter(name='ymin', value='-5000', format='int')
116 ##opObj10.addParameter(name='ymax', value='8500', format='int')
117 #opObj11.addParameter(name='save', value=figpath, format='str')
118 #opObj11.addParameter(name='save_period', value=10, format='int')
119
120 #opObj10 = procUnitConfObjA.addOperation(name='setH0')
121 #opObj10.addParameter(name='h0', value='-5000', format='float')
122
123 #opObj11 = procUnitConfObjA.addOperation(name='filterByHeights')
124 #opObj11.addParameter(name='window', value='1', format='int')
125
126 #codigo='1,1,-1,1,1,-1,1,-1,-1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,-1,1,1,1,1,-1,-1,-1'
127 #opObj11 = procUnitConfObjSousy.addOperation(name='Decoder', optype='other')
128 #opObj11.addParameter(name='code', value=codigo, formatyesterday='floatlist')
129 #opObj11.addParameter(name='nCode', value='1', format='int')
130 #opObj11.addParameter(name='nBaud', value='28', format='int')
131
132 #opObj11 = procUnitConfObjA.addOperation(name='CohInt', optype='other')
133 #opObj11.addParameter(name='n', value='100', format='int')
134
135 #######################################################################
136 ########## OPERACIONES ParametersProc########################
137 #######################################################################
138
139 procUnitConfObjB= controllerObj.addProcUnit(datatype='ParametersProc',inputId=procUnitConfObjA.getId())
140 opObj10 = procUnitConfObjB.addOperation(name='HDFWriter')
141 opObj10.addParameter(name='path',value=path_pp)
142 #opObj10.addParameter(name='mode',value=0)
143 opObj10.addParameter(name='blocksPerFile',value='100',format='int')
144 opObj10.addParameter(name='metadataList',value='utctimeInit,timeZone,paramInterval,profileIndex,channelList,heightList,flagDataAsBlock',format='list')
145 opObj10.addParameter(name='dataList',value='dataPP_POW,dataPP_DOP,utctime',format='list')#,format='list'
146
147 controllerObj.start()
Show file before | Show file after | Hide comments
@@ -0,0 +1,115
1 #!python
2 '''
3 '''
4
5 import os, sys
6 import datetime
7 import time
8
9 #path = os.path.dirname(os.getcwd())
10 #path = os.path.dirname(path)
11 #sys.path.insert(0, path)
12
13 from schainpy.controller import Project
14
15 desc = "USRP_test"
16 filename = "USRP_processing.xml"
17 controllerObj = Project()
18 controllerObj.setup(id = '191', name='Test_USRP', description=desc)
19
20 ############## USED TO PLOT IQ VOLTAGE, POWER AND SPECTRA #############
21
22 #######################################################################
23 ######PATH DE LECTURA, ESCRITURA, GRAFICOS Y ENVIO WEB#################
24 #######################################################################
25 #path = '/media/data/data/vientos/57.2063km/echoes/NCO_Woodman'
26 #path = '/DATA_RM/TEST_INTEGRACION'
27 #path = '/DATA_RM/TEST_ONLINE'
28 #path = '/DATA_RM/TEST_INTEGRACION/ADQ_OFFLINE/'
29 # ULTIMO TEST 22 DE SEPTIEMBRE
30 path = '/DATA_RM/USRP_22'
31 #path_pp = '/DATA_RM/TEST_HDF5'
32 # UTIMO TEST 22 DE SEPTIEMBRE
33 path_pp = '/DATA_RM/TEST_HDF5_PP_22'
34 ######################################################
35 ##### OJO TENER EN CUENTA EL n= para el Pulse Pair ###
36 ######################################################
37 ######## BUSCAMOS EL numero de IPP equivalente 1Β°#####
38 ######## Sea V la velocidad del Pedestal en Β°/seg#####
39 ######## 1Β° sera Recorrido en un tiempo de 1/V ######
40 ######## IPP del Radar 400 useg --> 60 Km ############
41 ######## n = 1/(V*IPP) , NUMERO DE IPP #############
42 ######## n = 1/(V*IPP) #############################
43 V=2
44 IPP=400*1e-6
45 n= 1/(V*IPP)
46 print("n numero de Perfiles a procesar con Pulse Pair: ", n)
47
48
49
50
51 figpath = '/home/soporte/Pictures/TEST_INTEGRACION_IMG'
52 #remotefolder = "/home/wmaster/graficos"
53 #######################################################################
54 ################# RANGO DE PLOTEO######################################
55 #######################################################################
56 dBmin = '-5'
57 dBmax = '20'
58 xmin = '0'
59 xmax ='24'
60 ymin = '0'
61 ymax = '600'
62 #######################################################################
63 ########################FECHA##########################################
64 #######################################################################
65 str = datetime.date.today()
66 today = str.strftime("%Y/%m/%d")
67 str2 = str - datetime.timedelta(days=1)
68 yesterday = str2.strftime("%Y/%m/%d")
69 #######################################################################
70 ######################## UNIDAD DE LECTURA#############################
71 #######################################################################
72 readUnitConfObj = controllerObj.addReadUnit(datatype='DigitalRFReader',
73 path=path,
74 startDate="2021/01/01",#today,
75 endDate="2021/12/30",#today,
76 startTime='00:00:00',
77 endTime='23:59:59',
78 delay=0,
79 #set=0,
80 online=0,
81 walk=1,
82 ippKm = 60)
83
84 opObj11 = readUnitConfObj.addOperation(name='printInfo')
85 #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
86 #######################################################################
87 ################ OPERACIONES DOMINIO DEL TIEMPO########################
88 #######################################################################
89
90 procUnitConfObjA = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
91
92 #
93 # codigo64='1,1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,1,1,1,0,1,1,0,1,0,0,0,1,1,1,0,1,1,1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,1,0,1,1,1,0,0,0,1,0,'+\
94 # '1,1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,1,1,1,0,1,1,0,1,0,0,0,1,1,1,0,1,0,0,0,1,0,0,1,0,0,0,0,1,1,1,0,1,1,1,1,0,1,1,0,1,0,0,0,1,1,1,0,1'
95
96 #opObj11 = procUnitConfObjA.addOperation(name='setRadarFrequency')
97 #opObj11.addParameter(name='frequency', value='70312500')
98 opObj11 = procUnitConfObjA.addOperation(name='PulsePair', optype='other')
99 opObj11.addParameter(name='n', value=int(n), format='int')#10 VOY A USAR 250 DADO QUE LA VELOCIDAD ES 10 GRADOS
100 opObj11.addParameter(name='removeDC', value=1, format='int')
101
102
103 #######################################################################
104 ########## OPERACIONES ParametersProc########################
105 #######################################################################
106
107 procUnitConfObjB= controllerObj.addProcUnit(datatype='ParametersProc',inputId=procUnitConfObjA.getId())
108 opObj10 = procUnitConfObjB.addOperation(name='HDFWriter')
109 opObj10.addParameter(name='path',value=path_pp)
110 #opObj10.addParameter(name='mode',value=0)
111 opObj10.addParameter(name='blocksPerFile',value='100',format='int')
112 opObj10.addParameter(name='metadataList',value='utctimeInit,timeZone,paramInterval,profileIndex,channelList,heightList,flagDataAsBlock',format='list')
113 opObj10.addParameter(name='dataList',value='dataPP_POW,dataPP_DOP,utctime',format='list')#,format='list'
114
115 controllerObj.start()
Show file before | Show file after | Hide comments
@@ -0,0 +1,88
1 #!python
2
3 import os, sys
4 import datetime
5 import time
6
7 from schainpy.controller import Project
8
9 desc = "USRP_test"
10 filename = "USRP_processing.xml"
11 controllerObj = Project()
12 controllerObj.setup(id = '191', name='Test_USRP', description=desc)
13
14 ############## USED TO PLOT IQ VOLTAGE, POWER AND SPECTRA #############
15
16 #######################################################################
17 ######PATH DE LECTURA, ESCRITURA, GRAFICOS Y ENVIO WEB#################
18 #######################################################################
19
20 path = '/DATA_RM/TEST_19OCTUBRE/10MHZ'
21 #path_pp = '/DATA_RM/TEST_HDF5'
22 path_pp = '/DATA_RM/TEST_HDF5_19OCT'
23
24 #######################################################################
25 ################# RANGO DE PLOTEO######################################
26 #######################################################################
27 dBmin = '-5'
28 dBmax = '20'
29 xmin = '0'
30 xmax ='24'
31 ymin = '0'
32 ymax = '600'
33 #######################################################################
34 ########################FECHA##########################################
35 #######################################################################
36 str = datetime.date.today()
37 today = str.strftime("%Y/%m/%d")
38 str2 = str - datetime.timedelta(days=1)
39 yesterday = str2.strftime("%Y/%m/%d")
40 #######################################################################
41 ######################## UNIDAD DE LECTURA#############################
42 #######################################################################
43 readUnitConfObj = controllerObj.addReadUnit(datatype='DigitalRFReader',
44 path=path,
45 startDate="2021/01/01",#today,
46 endDate="2021/12/30",#today,
47 startTime='00:00:00',
48 endTime='23:59:59',
49 delay=0,
50 #set=0,
51 online=0,
52 walk=1,
53 ippKm = 60)
54
55 opObj11 = readUnitConfObj.addOperation(name='printInfo')
56 #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
57 #######################################################################
58 ################ OPERACIONES DOMINIO DEL TIEMPO########################
59 #######################################################################
60 V=10 # aca se coloca la velocidad
61 IPP=400*1e-6
62 n= int(1/(V*IPP))
63 print("n numero de Perfiles a procesar con nFFTPoints ", n)
64
65 procUnitConfObjA = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
66
67 procUnitConfObjB = controllerObj.addProcUnit(datatype='SpectraProc', inputId=procUnitConfObjA.getId())
68 procUnitConfObjB.addParameter(name='nFFTPoints', value=n, format='int')
69 procUnitConfObjB.addParameter(name='nProfiles' , value=n, format='int')
70
71 #######################################################################
72 ########## OPERACIONES ParametersProc########################
73 #######################################################################
74
75 procUnitConfObjC= controllerObj.addProcUnit(datatype='ParametersProc',inputId=procUnitConfObjB.getId())
76
77 procUnitConfObjC.addOperation(name='SpectralMoments')
78
79 opObj10 = procUnitConfObjC.addOperation(name='HDFWriter')
80 opObj10.addParameter(name='path',value=path_pp)
81 #opObj10.addParameter(name='mode',value=0)
82 opObj10.addParameter(name='blocksPerFile',value='100',format='int')
83 #opObj10.addParameter(name='metadataList',value='utctimeInit,heightList,nIncohInt,nCohInt,nProfiles,channelList',format='list')#profileIndex
84 opObj10.addParameter(name='metadataList',value='utctimeInit,heightList,nIncohInt,nCohInt,nProfiles,channelList',format='list')#profileIndex
85
86 opObj10.addParameter(name='dataList',value='data_pow,data_dop,utctime',format='list')#,format='list'
87
88 controllerObj.start()
Show file before | Show file after | Hide comments
@@ -0,0 +1,36
1 import numpy as np
2 import matplotlib.pyplot as pl
3 import wradlib
4 import warnings
5 #export WRADLIB_DATA=/path/to/wradlib-data
6 warnings.filterwarnings('ignore')
7 '''
8 try:
9 get_ipython().magic('matplotlib inline')
10 except:
11 pl.ion()
12 '''
13 filename = wradlib.util.get_wradlib_data_file("/home/soporte/Downloads/raa00-dx_10908-0806021735-fbg---bin.gz")
14 img, meta = wradlib.io.read_dx(filename)
15 print("Shape of polar array: %r\n" % (img.shape,))
16 print("Some meta data of the DX file:")
17 print("\tdatetime: %r" % (meta["datetime"],))
18 print("\tRadar ID: %s" % (meta["radarid"],))
19
20 img[200:250,:]= np.ones([50,img.shape[1]])*np.nan
21
22 img[300:360,:]= np.ones([60,img.shape[1]])*np.nan
23
24 cgax, pm= wradlib.vis.plot_ppi(img)
25 txt = pl.title('Simple PPI')
26 print("coordenada angular",img[:,0],len(img[:,0]))
27 print("COORDENADA 0",img[0],len(img[0]))
28 cbar = pl.gcf().colorbar(pm, pad=0.075)
29
30 #r = np.arange(40, 80)
31 #az = np.arange(200, 250)
32 #ax, pm = wradlib.vis.plot_ppi(img[200:250, 40:80], r, az, autoext=False)
33 #ax, pm = wradlib.vis.plot_ppi(img[200:250, 40:80], r, az)
34
35 #txt = pl.title('Sector PPI')
36 pl.show()
Show file before | Show file after | Hide comments
@@ -0,0 +1,17
1 # Ing. AVP
2 # 01/11/2021
3 # ARCHIVO DE LECTURA
4 import os, sys
5 import datetime
6 import time
7 from schainpy.controller import Project
8 print("----[Setup]-RadarMeteorologico--------")
9 Vel = 6
10 modo_proc = 1
11 path = "/DATA_RM/10"
12 path_ped = "/DATA_RM/TEST_PEDESTAL/P20211111-173856"
13 print("----[OPCIONES]------------------------")
14 op_plot = 0
15 op_integration = 0
16 op_save = 0
17 op_plot_spec = 0
Show file before | Show file after | Hide comments
@@ -1,517 +1,605
1 1 import os
2 2 import datetime
3 3 import numpy
4 4
5 5 from schainpy.model.graphics.jroplot_base import Plot, plt
6 6 from schainpy.model.graphics.jroplot_spectra import SpectraPlot, RTIPlot, CoherencePlot, SpectraCutPlot
7 7 from schainpy.utils import log
8 8 # libreria wradlib
9 9 import wradlib as wrl
10 10
11 11 EARTH_RADIUS = 6.3710e3
12 12
13 13
14 14 def ll2xy(lat1, lon1, lat2, lon2):
15 15
16 16 p = 0.017453292519943295
17 17 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
18 18 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
19 19 r = 12742 * numpy.arcsin(numpy.sqrt(a))
20 20 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
21 21 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
22 22 theta = -theta + numpy.pi/2
23 23 return r*numpy.cos(theta), r*numpy.sin(theta)
24 24
25 25
26 26 def km2deg(km):
27 27 '''
28 28 Convert distance in km to degrees
29 29 '''
30 30
31 31 return numpy.rad2deg(km/EARTH_RADIUS)
32 32
33 33
34 34
35 35 class SpectralMomentsPlot(SpectraPlot):
36 36 '''
37 37 Plot for Spectral Moments
38 38 '''
39 39 CODE = 'spc_moments'
40 40 # colormap = 'jet'
41 41 # plot_type = 'pcolor'
42 42
43 43 class DobleGaussianPlot(SpectraPlot):
44 44 '''
45 45 Plot for Double Gaussian Plot
46 46 '''
47 47 CODE = 'gaussian_fit'
48 48 # colormap = 'jet'
49 49 # plot_type = 'pcolor'
50 50
51 51 class DoubleGaussianSpectraCutPlot(SpectraCutPlot):
52 52 '''
53 53 Plot SpectraCut with Double Gaussian Fit
54 54 '''
55 55 CODE = 'cut_gaussian_fit'
56 56
57 57 class SnrPlot(RTIPlot):
58 58 '''
59 59 Plot for SNR Data
60 60 '''
61 61
62 62 CODE = 'snr'
63 63 colormap = 'jet'
64 64
65 65 def update(self, dataOut):
66 66
67 67 data = {
68 68 'snr': 10*numpy.log10(dataOut.data_snr)
69 69 }
70 70
71 71 return data, {}
72 72
73 73 class DopplerPlot(RTIPlot):
74 74 '''
75 75 Plot for DOPPLER Data (1st moment)
76 76 '''
77 77
78 78 CODE = 'dop'
79 79 colormap = 'jet'
80 80
81 81 def update(self, dataOut):
82 82
83 83 data = {
84 84 'dop': 10*numpy.log10(dataOut.data_dop)
85 85 }
86 86
87 87 return data, {}
88 88
89 89 class PowerPlot(RTIPlot):
90 90 '''
91 91 Plot for Power Data (0 moment)
92 92 '''
93 93
94 94 CODE = 'pow'
95 95 colormap = 'jet'
96 96
97 97 def update(self, dataOut):
98 98 data = {
99 99 'pow': 10*numpy.log10(dataOut.data_pow/dataOut.normFactor)
100 100 }
101 101 return data, {}
102 102
103 103 class SpectralWidthPlot(RTIPlot):
104 104 '''
105 105 Plot for Spectral Width Data (2nd moment)
106 106 '''
107 107
108 108 CODE = 'width'
109 109 colormap = 'jet'
110 110
111 111 def update(self, dataOut):
112 112
113 113 data = {
114 114 'width': dataOut.data_width
115 115 }
116 116
117 117 return data, {}
118 118
119 119 class SkyMapPlot(Plot):
120 120 '''
121 121 Plot for meteors detection data
122 122 '''
123 123
124 124 CODE = 'param'
125 125
126 126 def setup(self):
127 127
128 128 self.ncols = 1
129 129 self.nrows = 1
130 130 self.width = 7.2
131 131 self.height = 7.2
132 132 self.nplots = 1
133 133 self.xlabel = 'Zonal Zenith Angle (deg)'
134 134 self.ylabel = 'Meridional Zenith Angle (deg)'
135 135 self.polar = True
136 136 self.ymin = -180
137 137 self.ymax = 180
138 138 self.colorbar = False
139 139
140 140 def plot(self):
141 141
142 142 arrayParameters = numpy.concatenate(self.data['param'])
143 143 error = arrayParameters[:, -1]
144 144 indValid = numpy.where(error == 0)[0]
145 145 finalMeteor = arrayParameters[indValid, :]
146 146 finalAzimuth = finalMeteor[:, 3]
147 147 finalZenith = finalMeteor[:, 4]
148 148
149 149 x = finalAzimuth * numpy.pi / 180
150 150 y = finalZenith
151 151
152 152 ax = self.axes[0]
153 153
154 154 if ax.firsttime:
155 155 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
156 156 else:
157 157 ax.plot.set_data(x, y)
158 158
159 159 dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
160 160 dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
161 161 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
162 162 dt2,
163 163 len(x))
164 164 self.titles[0] = title
165 165
166 166
167 167 class GenericRTIPlot(Plot):
168 168 '''
169 169 Plot for data_xxxx object
170 170 '''
171 171
172 172 CODE = 'param'
173 173 colormap = 'viridis'
174 174 plot_type = 'pcolorbuffer'
175 175
176 176 def setup(self):
177 177 self.xaxis = 'time'
178 178 self.ncols = 1
179 179 self.nrows = self.data.shape('param')[0]
180 180 self.nplots = self.nrows
181 181 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
182 182
183 183 if not self.xlabel:
184 184 self.xlabel = 'Time'
185 185
186 186 self.ylabel = 'Range [km]'
187 187 if not self.titles:
188 188 self.titles = ['Param {}'.format(x) for x in range(self.nrows)]
189 189
190 190 def update(self, dataOut):
191 191
192 192 data = {
193 193 'param' : numpy.concatenate([getattr(dataOut, attr) for attr in self.attr_data], axis=0)
194 194 }
195 195
196 196 meta = {}
197 197
198 198 return data, meta
199 199
200 200 def plot(self):
201 201 # self.data.normalize_heights()
202 202 self.x = self.data.times
203 203 self.y = self.data.yrange
204 204 self.z = self.data['param']
205 205 self.z = 10*numpy.log10(self.z)
206 206 self.z = numpy.ma.masked_invalid(self.z)
207 207
208 208 if self.decimation is None:
209 209 x, y, z = self.fill_gaps(self.x, self.y, self.z)
210 210 else:
211 211 x, y, z = self.fill_gaps(*self.decimate())
212 212
213 213 for n, ax in enumerate(self.axes):
214 214
215 215 self.zmax = self.zmax if self.zmax is not None else numpy.max(
216 216 self.z[n])
217 217 self.zmin = self.zmin if self.zmin is not None else numpy.min(
218 218 self.z[n])
219 219
220 220 if ax.firsttime:
221 221 if self.zlimits is not None:
222 222 self.zmin, self.zmax = self.zlimits[n]
223 223
224 224 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
225 225 vmin=self.zmin,
226 226 vmax=self.zmax,
227 227 cmap=self.cmaps[n]
228 228 )
229 229 else:
230 230 if self.zlimits is not None:
231 231 self.zmin, self.zmax = self.zlimits[n]
232 232 ax.collections.remove(ax.collections[0])
233 233 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
234 234 vmin=self.zmin,
235 235 vmax=self.zmax,
236 236 cmap=self.cmaps[n]
237 237 )
238 238
239 239
240 240 class PolarMapPlot(Plot):
241 241 '''
242 242 Plot for weather radar
243 243 '''
244 244
245 245 CODE = 'param'
246 246 colormap = 'seismic'
247 247
248 248 def setup(self):
249 249 self.ncols = 1
250 250 self.nrows = 1
251 251 self.width = 9
252 252 self.height = 8
253 253 self.mode = self.data.meta['mode']
254 254 if self.channels is not None:
255 255 self.nplots = len(self.channels)
256 256 self.nrows = len(self.channels)
257 257 else:
258 258 self.nplots = self.data.shape(self.CODE)[0]
259 259 self.nrows = self.nplots
260 260 self.channels = list(range(self.nplots))
261 261 if self.mode == 'E':
262 262 self.xlabel = 'Longitude'
263 263 self.ylabel = 'Latitude'
264 264 else:
265 265 self.xlabel = 'Range (km)'
266 266 self.ylabel = 'Height (km)'
267 267 self.bgcolor = 'white'
268 268 self.cb_labels = self.data.meta['units']
269 269 self.lat = self.data.meta['latitude']
270 270 self.lon = self.data.meta['longitude']
271 271 self.xmin, self.xmax = float(
272 272 km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
273 273 self.ymin, self.ymax = float(
274 274 km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
275 275 # self.polar = True
276 276
277 277 def plot(self):
278 278
279 279 for n, ax in enumerate(self.axes):
280 280 data = self.data['param'][self.channels[n]]
281 281
282 282 zeniths = numpy.linspace(
283 283 0, self.data.meta['max_range'], data.shape[1])
284 284 if self.mode == 'E':
285 285 azimuths = -numpy.radians(self.data.yrange)+numpy.pi/2
286 286 r, theta = numpy.meshgrid(zeniths, azimuths)
287 287 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
288 288 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
289 289 x = km2deg(x) + self.lon
290 290 y = km2deg(y) + self.lat
291 291 else:
292 292 azimuths = numpy.radians(self.data.yrange)
293 293 r, theta = numpy.meshgrid(zeniths, azimuths)
294 294 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
295 295 self.y = zeniths
296 296
297 297 if ax.firsttime:
298 298 if self.zlimits is not None:
299 299 self.zmin, self.zmax = self.zlimits[n]
300 300 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
301 301 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
302 302 vmin=self.zmin,
303 303 vmax=self.zmax,
304 304 cmap=self.cmaps[n])
305 305 else:
306 306 if self.zlimits is not None:
307 307 self.zmin, self.zmax = self.zlimits[n]
308 308 ax.collections.remove(ax.collections[0])
309 309 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
310 310 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
311 311 vmin=self.zmin,
312 312 vmax=self.zmax,
313 313 cmap=self.cmaps[n])
314 314
315 315 if self.mode == 'A':
316 316 continue
317 317
318 318 # plot district names
319 319 f = open('/data/workspace/schain_scripts/distrito.csv')
320 320 for line in f:
321 321 label, lon, lat = [s.strip() for s in line.split(',') if s]
322 322 lat = float(lat)
323 323 lon = float(lon)
324 324 # ax.plot(lon, lat, '.b', ms=2)
325 325 ax.text(lon, lat, label.decode('utf8'), ha='center',
326 326 va='bottom', size='8', color='black')
327 327
328 328 # plot limites
329 329 limites = []
330 330 tmp = []
331 331 for line in open('/data/workspace/schain_scripts/lima.csv'):
332 332 if '#' in line:
333 333 if tmp:
334 334 limites.append(tmp)
335 335 tmp = []
336 336 continue
337 337 values = line.strip().split(',')
338 338 tmp.append((float(values[0]), float(values[1])))
339 339 for points in limites:
340 340 ax.add_patch(
341 341 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
342 342
343 343 # plot Cuencas
344 344 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
345 345 f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
346 346 values = [line.strip().split(',') for line in f]
347 347 points = [(float(s[0]), float(s[1])) for s in values]
348 348 ax.add_patch(Polygon(points, ec='b', fc='none'))
349 349
350 350 # plot grid
351 351 for r in (15, 30, 45, 60):
352 352 ax.add_artist(plt.Circle((self.lon, self.lat),
353 353 km2deg(r), color='0.6', fill=False, lw=0.2))
354 354 ax.text(
355 355 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
356 356 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
357 357 '{}km'.format(r),
358 358 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
359 359
360 360 if self.mode == 'E':
361 361 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
362 362 label = 'E{:02d}'.format(int(self.data.meta['elevation']))
363 363 else:
364 364 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
365 365 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
366 366
367 367 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
368 368 self.titles = ['{} {}'.format(
369 369 self.data.parameters[x], title) for x in self.channels]
370 370
371 371 class WeatherPlot(Plot):
372 372 CODE = 'weather'
373 373 plot_name = 'weather'
374 374 plot_type = 'ppistyle'
375 375 buffering = False
376 376
377 377 def setup(self):
378 378 self.ncols = 1
379 379 self.nrows = 1
380 380 self.nplots= 1
381 381 self.ylabel= 'Range [Km]'
382 382 self.titles= ['Weather']
383 383 self.colorbar=False
384 384 self.width =8
385 385 self.height =8
386 386 self.ini =0
387 387 self.len_azi =0
388 388 self.buffer_ini = None
389 389 self.buffer_azi = None
390 390 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08})
391 391 self.flag =0
392 392 self.indicador= 0
393 self.last_data_azi = None
394 self.val_mean = None
393 395
394 396 def update(self, dataOut):
395 397
396 398 data = {}
397 399 meta = {}
398 400 if hasattr(dataOut, 'dataPP_POWER'):
399 401 factor = 1
400 402
401 403 if hasattr(dataOut, 'nFFTPoints'):
402 404 factor = dataOut.normFactor
403 405
404 406 ####print("factor",factor)
405 data['weather'] = 10*numpy.log10(dataOut.data_360[0]/(factor))
406 print("weather",data['weather'])
407 data['weather'] = 10*numpy.log10(dataOut.data_360[1]/(factor))
408 ####print("weather",data['weather'])
407 409 data['azi'] = dataOut.data_azi
408 410 return data, meta
409 411
412 def analizeDATA(self,data_azi):
413 list1 = []
414 list2 = []
415 dat = data_azi
416 for i in reversed(range(1,len(dat))):
417 if dat[i]>dat[i-1]:
418 diff = int(dat[i])-int(dat[i-1])
419 else:
420 diff = 360+int(dat[i])-int(dat[i-1])
421 if diff > 1:
422 list1.append(i-1)
423 list2.append(diff-1)
424 return list1,list2
425
426 def fixDATANEW(self,data_azi,data_weather):
427 list1,list2 = self.analizeDATA(data_azi)
428 if len(list1)== 0:
429 return data_azi,data_weather
430 else:
431 resize = 0
432 for i in range(len(list2)):
433 resize= resize + list2[i]
434 new_data_azi = numpy.resize(data_azi,resize)
435 new_data_weather= numpy.resize(date_weather,resize)
436
437 for i in range(len(list2)):
438 j=0
439 position=list1[i]+1
440 for j in range(list2[i]):
441 new_data_azi[position+j]=new_data_azi[position+j-1]+1
442
443 return new_data_azi
444
445 def fixDATA(self,data_azi):
446 data=data_azi
447 for i in range(len(data)):
448 if numpy.isnan(data[i]):
449 data[i]=data[i-1]+1
450 return data
451
452 def replaceNAN(self,data_weather,data_azi,val):
453 print("----------------activeNEWFUNCTION")
454 data= data_azi
455 data_T= data_weather
456 print("data_azi",data_azi)
457 print("VAL:",val)
458 print("SHAPE",data_T.shape)
459 for i in range(len(data)):
460 if numpy.isnan(data[i]):
461 print("NAN")
462 data_T[i,:]=numpy.ones(data_T.shape[1])*val
463 #data_T[i,:]=numpy.ones(data_T.shape[1])*numpy.nan
464 return data_T
465
410 466 def const_ploteo(self,data_weather,data_azi,step,res):
411 467 if self.ini==0:
412 468 #------- AZIMUTH
413 469 n = (360/res)-len(data_azi)
414 start = data_azi[-1] + res
415 end = data_azi[0] - res
470 ##### new
471 data_azi_old = data_azi
472 data_azi_new = self.fixDATA(data_azi)
473 #ata_azi_new = self.fixDATANEW(data_azi)
474
475 start = data_azi_new[-1] + res
476 end = data_azi_new[0] - res
477 ##### new
478 self.last_data_azi = end
416 479 if start>end:
417 480 end = end + 360
418 481 azi_vacia = numpy.linspace(start,end,int(n))
419 482 azi_vacia = numpy.where(azi_vacia>360,azi_vacia-360,azi_vacia)
420 data_azi = numpy.hstack((data_azi,azi_vacia))
483 data_azi = numpy.hstack((data_azi_new,azi_vacia))
421 484 # RADAR
422 val_mean = numpy.mean(data_weather[:,0])
485 val_mean = numpy.mean(data_weather[:,-1])
486 self.val_mean = val_mean
423 487 data_weather_cmp = numpy.ones([(360-data_weather.shape[0]),data_weather.shape[1]])*val_mean
488 data_weather = self.replaceNAN(data_weather=data_weather,data_azi=data_azi_old,val=self.val_mean)
424 489 data_weather = numpy.vstack((data_weather,data_weather_cmp))
425 490 else:
426 491 # azimuth
427 492 flag=0
428 493 start_azi = self.res_azi[0]
494 #### new
495 data_azi_old = data_azi
496 ### weather ###
497 data_weather = self.replaceNAN(data_weather=data_weather,data_azi=data_azi_old,val=self.val_mean)
498
499 if numpy.isnan(data_azi[0]):
500 data_azi[0]=self.last_data_azi+1
501 data_azi = self.fixDATA(data_azi)
502 ####
503
429 504 start = data_azi[0]
430 505 end = data_azi[-1]
431 print("start",start)
432 print("end",end)
506 self.last_data_azi= end
507 ####print("start",start)
508 ####print("end",end)
433 509 if start< start_azi:
434 510 start = start +360
435 511 if end <start_azi:
436 512 end = end +360
437 513
438 print("start",start)
439 print("end",end)
514 ####print("start",start)
515 ####print("end",end)
440 516 #### AQUI SERA LA MAGIA
441 517 pos_ini = int((start-start_azi)/res)
442 518 len_azi = len(data_azi)
443 519 if (360-pos_ini)<len_azi:
444 520 if pos_ini+1==360:
445 521 pos_ini=0
446 522 else:
447 523 flag=1
448 524 dif= 360-pos_ini
449 525 comp= len_azi-dif
450 526
451 print(pos_ini)
452 print(len_azi)
453 print("shape",self.res_azi.shape)
527 #-----------------
528 ####print(pos_ini)
529 ####print(len_azi)
530 ####print("shape",self.res_azi.shape)
454 531 if flag==0:
455 532 # AZIMUTH
456 533 self.res_azi[pos_ini:pos_ini+len_azi] = data_azi
457 534 # RADAR
458 535 self.res_weather[pos_ini:pos_ini+len_azi,:] = data_weather
459 536 else:
460 537 # AZIMUTH
461 538 self.res_azi[pos_ini:pos_ini+dif] = data_azi[0:dif]
462 539 self.res_azi[0:comp] = data_azi[dif:]
463 540 # RADAR
464 541 self.res_weather[pos_ini:pos_ini+dif,:] = data_weather[0:dif,:]
465 542 self.res_weather[0:comp,:] = data_weather[dif:,:]
466 543 flag=0
467 544 data_azi = self.res_azi
468 545 data_weather = self.res_weather
469 546
470 547 return data_weather,data_azi
471 548
472 549 def plot(self):
473 print("--------------------------------------",self.ini,"-----------------------------------")
550 #print("--------------------------------------",self.ini,"-----------------------------------")
474 551 #numpy.set_printoptions(suppress=True)
475 #print(self.data.times)
476 thisDatetime = datetime.datetime.utcfromtimestamp(self.data.times[-1])
552 ####print("times: ",self.data.times)
553 thisDatetime = datetime.datetime.utcfromtimestamp(self.data.times[-1]).strftime('%Y-%m-%d %H:%M:%S')
554 #print("times: ",thisDatetime)
477 555 data = self.data[-1]
478 # ALTURA altura_tmp_h
479 altura_h = (data['weather'].shape[1])/10.0
480 stoprange = float(altura_h*1.5)#stoprange = float(33*1.5) por ahora 400
481 rangestep = float(0.15)
482 r = numpy.arange(0, stoprange, rangestep)
556 ####ALTURA altura_tmp_h
557 ###print("Y RANGES",self.data.yrange,len(self.data.yrange))
558 ###altura_h = (data['weather'].shape[1])/10.0
559 ###stoprange = float(altura_h*0.3)#stoprange = float(33*1.5) por ahora 400
560 ###rangestep = float(0.03)
561 ###r = numpy.arange(0, stoprange, rangestep)
562 ###print("r",r,len(r))
563 #-----------------------------update----------------------
564 r= self.data.yrange
565 delta_height = r[1]-r[0]
566 #print("1",r)
567 r_mask= numpy.where(r>=0)[0]
568 r = numpy.arange(len(r_mask))*delta_height
569 #print("2",r)
483 570 self.y = 2*r
571 ######self.y = self.data.yrange
484 572 # RADAR
485 573 #data_weather = data['weather']
486 574 # PEDESTAL
487 575 #data_azi = data['azi']
488 576 res = 1
489 577 # STEP
490 578 step = (360/(res*data['weather'].shape[0]))
491 579 #print("shape wr_data", wr_data.shape)
492 580 #print("shape wr_azi",wr_azi.shape)
493 581 #print("step",step)
494 print("Time---->",self.data.times[-1],thisDatetime)
495 #print("alturas", len(self.y))
496 self.res_weather, self.res_azi = self.const_ploteo(data_weather=data['weather'],data_azi=data['azi'],step=step,res=res)
582 ####print("Time---->",self.data.times[-1],thisDatetime)
583 #print("alturas", len(self.y))numpy.where(r>=0)
584 self.res_weather, self.res_azi = self.const_ploteo(data_weather=data['weather'][:,r_mask],data_azi=data['azi'],step=step,res=res)
497 585 #numpy.set_printoptions(suppress=True)
498 586 #print("resultado",self.res_azi)
499 ##########################################################
587 ###########################/DATA_RM/10_tmp/ch0###############################
500 588 ################# PLOTEO ###################
501 589 ##########################################################
502 590
503 591 for i,ax in enumerate(self.axes):
504 592 if ax.firsttime:
505 593 plt.clf()
506 cgax, pm = wrl.vis.plot_ppi(self.res_weather,r=r,az=self.res_azi,fig=self.figures[0], proj='cg', vmin=1, vmax=60)
594 cgax, pm = wrl.vis.plot_ppi(self.res_weather,r=r,az=self.res_azi,fig=self.figures[0], proj='cg', vmin=8, vmax=35)
507 595 else:
508 596 plt.clf()
509 cgax, pm = wrl.vis.plot_ppi(self.res_weather,r=r,az=self.res_azi,fig=self.figures[0], proj='cg', vmin=1, vmax=60)
597 cgax, pm = wrl.vis.plot_ppi(self.res_weather,r=r,az=self.res_azi,fig=self.figures[0], proj='cg', vmin=8, vmax=35)
510 598 caax = cgax.parasites[0]
511 599 paax = cgax.parasites[1]
512 600 cbar = plt.gcf().colorbar(pm, pad=0.075)
513 601 caax.set_xlabel('x_range [km]')
514 602 caax.set_ylabel('y_range [km]')
515 plt.text(1.0, 1.05, 'azimuth '+str(thisDatetime)+"step"+str(self.ini), transform=caax.transAxes, va='bottom',ha='right')
603 plt.text(1.0, 1.05, 'azimuth '+str(thisDatetime)+" step "+str(self.ini), transform=caax.transAxes, va='bottom',ha='right')
516 604
517 605 self.ini= self.ini+1
Show file before | Show file after | Hide comments
@@ -1,798 +1,798
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 # SUBCHANNELS EN VEZ DE CHANNELS
7 7 # BENCHMARKS -> PROBLEMAS CON ARCHIVOS GRANDES -> INCONSTANTE EN EL TIEMPO
8 8 # ACTUALIZACION DE VERSION
9 9 # HEADERS
10 10 # MODULO DE ESCRITURA
11 11 # METADATA
12 12
13 13 import os
14 14 import time
15 15 import datetime
16 16 import numpy
17 17 import timeit
18 18 from fractions import Fraction
19 19 from time import time
20 20 from time import sleep
21 21
22 22 import schainpy.admin
23 23 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
24 24 from schainpy.model.data.jrodata import Voltage
25 25 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
26 26
27 27 import pickle
28 28 try:
29 29 import digital_rf
30 30 except:
31 31 pass
32 32
33 33
34 34 class DigitalRFReader(ProcessingUnit):
35 35 '''
36 36 classdocs
37 37 '''
38 38
39 39 def __init__(self):
40 40 '''
41 41 Constructor
42 42 '''
43 43
44 44 ProcessingUnit.__init__(self)
45 45
46 46 self.dataOut = Voltage()
47 47 self.__printInfo = True
48 48 self.__flagDiscontinuousBlock = False
49 49 self.__bufferIndex = 9999999
50 50 self.__codeType = 0
51 51 self.__ippKm = None
52 52 self.__nCode = None
53 53 self.__nBaud = None
54 54 self.__code = None
55 55 self.dtype = None
56 56 self.oldAverage = None
57 57 self.path = None
58 58
59 59 def close(self):
60 60 print('Average of writing to digital rf format is ', self.oldAverage * 1000)
61 61 return
62 62
63 63 def __getCurrentSecond(self):
64 64
65 65 return self.__thisUnixSample / self.__sample_rate
66 66
67 67 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
68 68
69 69 def __setFileHeader(self):
70 70 '''
71 71 In this method will be initialized every parameter of dataOut object (header, no data)
72 72 '''
73 73 ippSeconds = 1.0 * self.__nSamples / self.__sample_rate
74 74
75 75 nProfiles = 1.0 / ippSeconds # Number of profiles in one second
76 76
77 77 try:
78 78 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
79 79 self.__radarControllerHeader)
80 80 except:
81 81 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
82 82 txA=0,
83 83 txB=0,
84 84 nWindows=1,
85 85 nHeights=self.__nSamples,
86 86 firstHeight=self.__firstHeigth,
87 87 deltaHeight=self.__deltaHeigth,
88 88 codeType=self.__codeType,
89 89 nCode=self.__nCode, nBaud=self.__nBaud,
90 90 code=self.__code)
91 91
92 92 try:
93 93 self.dataOut.systemHeaderObj = SystemHeader(self.__systemHeader)
94 94 except:
95 95 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
96 96 nProfiles=nProfiles,
97 97 nChannels=len(
98 98 self.__channelList),
99 99 adcResolution=14)
100 100 self.dataOut.type = "Voltage"
101 101
102 102 self.dataOut.data = None
103 103
104 104 self.dataOut.dtype = self.dtype
105 105
106 106 # self.dataOut.nChannels = 0
107 107
108 108 # self.dataOut.nHeights = 0
109 109
110 110 self.dataOut.nProfiles = int(nProfiles)
111 111
112 112 self.dataOut.heightList = self.__firstHeigth + \
113 113 numpy.arange(self.__nSamples, dtype=numpy.float) * \
114 114 self.__deltaHeigth
115 115
116 116 #self.dataOut.channelList = list(range(self.__num_subchannels))
117 117 self.dataOut.channelList = list(range(len(self.__channelList)))
118 118 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
119 119
120 120 # self.dataOut.channelIndexList = None
121 121
122 122 self.dataOut.flagNoData = True
123 123
124 124 self.dataOut.flagDataAsBlock = False
125 125 # Set to TRUE if the data is discontinuous
126 126 self.dataOut.flagDiscontinuousBlock = False
127 127
128 128 self.dataOut.utctime = None
129 129
130 130 # timezone like jroheader, difference in minutes between UTC and localtime
131 131 self.dataOut.timeZone = self.__timezone / 60
132 132
133 133 self.dataOut.dstFlag = 0
134 134
135 135 self.dataOut.errorCount = 0
136 136
137 137 try:
138 138 self.dataOut.nCohInt = self.fixed_metadata_dict.get(
139 139 'nCohInt', self.nCohInt)
140 140
141 141 # asumo que la data esta decodificada
142 142 self.dataOut.flagDecodeData = self.fixed_metadata_dict.get(
143 143 'flagDecodeData', self.flagDecodeData)
144 144
145 145 # asumo que la data esta sin flip
146 146 self.dataOut.flagDeflipData = self.fixed_metadata_dict['flagDeflipData']
147 147
148 148 self.dataOut.flagShiftFFT = self.fixed_metadata_dict['flagShiftFFT']
149 149
150 150 self.dataOut.useLocalTime = self.fixed_metadata_dict['useLocalTime']
151 151 except:
152 152 pass
153 153
154 154 self.dataOut.ippSeconds = ippSeconds
155 155
156 156 # Time interval between profiles
157 157 # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
158 158
159 159 self.dataOut.frequency = self.__frequency
160 160
161 161 self.dataOut.realtime = self.__online
162 162
163 163 def findDatafiles(self, path, startDate=None, endDate=None):
164 164
165 165 if not os.path.isdir(path):
166 166 return []
167 167
168 168 try:
169 169 digitalReadObj = digital_rf.DigitalRFReader(
170 170 path, load_all_metadata=True)
171 171 except:
172 172 digitalReadObj = digital_rf.DigitalRFReader(path)
173 173
174 174 channelNameList = digitalReadObj.get_channels()
175 175
176 176 if not channelNameList:
177 177 return []
178 178
179 179 metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
180 180
181 181 sample_rate = metadata_dict['sample_rate'][0]
182 182
183 183 this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
184 184
185 185 try:
186 186 timezone = this_metadata_file['timezone'].value
187 187 except:
188 188 timezone = 0
189 189
190 190 startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(
191 191 channelNameList[0]) / sample_rate - timezone
192 192
193 193 startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
194 194 endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
195 195
196 196 if not startDate:
197 197 startDate = startDatetime.date()
198 198
199 199 if not endDate:
200 200 endDate = endDatatime.date()
201 201
202 202 dateList = []
203 203
204 204 thisDatetime = startDatetime
205 205
206 206 while(thisDatetime <= endDatatime):
207 207
208 208 thisDate = thisDatetime.date()
209 209
210 210 if thisDate < startDate:
211 211 continue
212 212
213 213 if thisDate > endDate:
214 214 break
215 215
216 216 dateList.append(thisDate)
217 217 thisDatetime += datetime.timedelta(1)
218 218
219 219 return dateList
220 220
221 221 def setup(self, path=None,
222 222 startDate=None,
223 223 endDate=None,
224 224 startTime=datetime.time(0, 0, 0),
225 225 endTime=datetime.time(23, 59, 59),
226 226 channelList=None,
227 227 nSamples=None,
228 228 online=False,
229 229 delay=60,
230 230 buffer_size=1024,
231 231 ippKm=None,
232 232 nCohInt=1,
233 233 nCode=1,
234 234 nBaud=1,
235 235 flagDecodeData=False,
236 236 code=numpy.ones((1, 1), dtype=numpy.int),
237 237 **kwargs):
238 238 '''
239 239 In this method we should set all initial parameters.
240 240
241 241 Inputs:
242 242 path
243 243 startDate
244 244 endDate
245 245 startTime
246 246 endTime
247 247 set
248 248 expLabel
249 249 ext
250 250 online
251 251 delay
252 252 '''
253 253 self.path = path
254 254 self.nCohInt = nCohInt
255 255 self.flagDecodeData = flagDecodeData
256 256 self.i = 0
257 257 if not os.path.isdir(path):
258 258 raise ValueError("[Reading] Directory %s does not exist" % path)
259 259
260 260 try:
261 261 self.digitalReadObj = digital_rf.DigitalRFReader(
262 262 path, load_all_metadata=True)
263 263 except:
264 264 self.digitalReadObj = digital_rf.DigitalRFReader(path)
265 265
266 266 channelNameList = self.digitalReadObj.get_channels()
267 267
268 268 if not channelNameList:
269 269 raise ValueError("[Reading] Directory %s does not have any files" % path)
270 270
271 271 if not channelList:
272 272 channelList = list(range(len(channelNameList)))
273 273
274 274 ########## Reading metadata ######################
275 275
276 276 top_properties = self.digitalReadObj.get_properties(
277 277 channelNameList[channelList[0]])
278 278
279 279 self.__num_subchannels = top_properties['num_subchannels']
280 280 self.__sample_rate = 1.0 * \
281 281 top_properties['sample_rate_numerator'] / \
282 282 top_properties['sample_rate_denominator']
283 283 # self.__samples_per_file = top_properties['samples_per_file'][0]
284 284 self.__deltaHeigth = 1e6 * 0.15 / self.__sample_rate # why 0.15?
285 285
286 286 this_metadata_file = self.digitalReadObj.get_digital_metadata(
287 287 channelNameList[channelList[0]])
288 288 metadata_bounds = this_metadata_file.get_bounds()
289 289 self.fixed_metadata_dict = this_metadata_file.read(
290 290 metadata_bounds[0])[metadata_bounds[0]] # GET FIRST HEADER
291 291
292 292 try:
293 293 self.__processingHeader = self.fixed_metadata_dict['processingHeader']
294 294 self.__radarControllerHeader = self.fixed_metadata_dict['radarControllerHeader']
295 295 self.__systemHeader = self.fixed_metadata_dict['systemHeader']
296 296 self.dtype = pickle.loads(self.fixed_metadata_dict['dtype'])
297 297 except:
298 298 pass
299 299
300 300 self.__frequency = None
301 301
302 302 self.__frequency = self.fixed_metadata_dict.get('frequency', 1)
303 303
304 304 self.__timezone = self.fixed_metadata_dict.get('timezone', 18000)
305 305
306 306 try:
307 307 nSamples = self.fixed_metadata_dict['nSamples']
308 308 except:
309 309 nSamples = None
310 310
311 311 self.__firstHeigth = 0
312 312
313 313 try:
314 314 codeType = self.__radarControllerHeader['codeType']
315 315 except:
316 316 codeType = 0
317 317
318 318 try:
319 319 if codeType:
320 320 nCode = self.__radarControllerHeader['nCode']
321 321 nBaud = self.__radarControllerHeader['nBaud']
322 322 code = self.__radarControllerHeader['code']
323 323 except:
324 324 pass
325 325
326 326 if not ippKm:
327 327 try:
328 328 # seconds to km
329 329 ippKm = self.__radarControllerHeader['ipp']
330 330 except:
331 331 ippKm = None
332 332 ####################################################
333 333 self.__ippKm = ippKm
334 334 startUTCSecond = None
335 335 endUTCSecond = None
336 336
337 337 if startDate:
338 338 startDatetime = datetime.datetime.combine(startDate, startTime)
339 339 startUTCSecond = (
340 340 startDatetime - datetime.datetime(1970, 1, 1)).total_seconds() + self.__timezone
341 341
342 342 if endDate:
343 343 endDatetime = datetime.datetime.combine(endDate, endTime)
344 344 endUTCSecond = (endDatetime - datetime.datetime(1970,
345 345 1, 1)).total_seconds() + self.__timezone
346 346
347 347
348 348 print(startUTCSecond,endUTCSecond)
349 349 start_index, end_index = self.digitalReadObj.get_bounds(
350 350 channelNameList[channelList[0]])
351 351
352 print("*****",start_index,end_index)
352 ##print("*****",start_index,end_index)
353 353 if not startUTCSecond:
354 354 startUTCSecond = start_index / self.__sample_rate
355 355
356 356 if start_index > startUTCSecond * self.__sample_rate:
357 357 startUTCSecond = start_index / self.__sample_rate
358 358
359 359 if not endUTCSecond:
360 360 endUTCSecond = end_index / self.__sample_rate
361 361
362 362 if end_index < endUTCSecond * self.__sample_rate:
363 363 endUTCSecond = end_index / self.__sample_rate
364 364 if not nSamples:
365 365 if not ippKm:
366 366 raise ValueError("[Reading] nSamples or ippKm should be defined")
367 367 nSamples = int(ippKm / (1e6 * 0.15 / self.__sample_rate))
368 368 channelBoundList = []
369 369 channelNameListFiltered = []
370 370
371 371 for thisIndexChannel in channelList:
372 372 thisChannelName = channelNameList[thisIndexChannel]
373 373 start_index, end_index = self.digitalReadObj.get_bounds(
374 374 thisChannelName)
375 375 channelBoundList.append((start_index, end_index))
376 376 channelNameListFiltered.append(thisChannelName)
377 377
378 378 self.profileIndex = 0
379 379 self.i = 0
380 380 self.__delay = delay
381 381
382 382 self.__codeType = codeType
383 383 self.__nCode = nCode
384 384 self.__nBaud = nBaud
385 385 self.__code = code
386 386
387 387 self.__datapath = path
388 388 self.__online = online
389 389 self.__channelList = channelList
390 390 self.__channelNameList = channelNameListFiltered
391 391 self.__channelBoundList = channelBoundList
392 392 self.__nSamples = nSamples
393 393 self.__samples_to_read = int(nSamples) # FIJO: AHORA 40
394 394 self.__nChannels = len(self.__channelList)
395 395
396 396 self.__startUTCSecond = startUTCSecond
397 397 self.__endUTCSecond = endUTCSecond
398 398
399 399 self.__timeInterval = 1.0 * self.__samples_to_read / \
400 400 self.__sample_rate # Time interval
401 401
402 402 if online:
403 403 # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
404 404 startUTCSecond = numpy.floor(endUTCSecond)
405 405
406 406 # por que en el otro metodo lo primero q se hace es sumar samplestoread
407 407 self.__thisUnixSample = int(startUTCSecond * self.__sample_rate) - self.__samples_to_read
408 408
409 409 #self.__data_buffer = numpy.zeros(
410 410 # (self.__num_subchannels, self.__samples_to_read), dtype=numpy.complex)
411 411 self.__data_buffer = numpy.zeros((int(len(channelList)), self.__samples_to_read), dtype=numpy.complex)
412 412
413 413
414 414 self.__setFileHeader()
415 415 self.isConfig = True
416 416
417 417 print("[Reading] Digital RF Data was found from %s to %s " % (
418 418 datetime.datetime.utcfromtimestamp(
419 419 self.__startUTCSecond - self.__timezone),
420 420 datetime.datetime.utcfromtimestamp(
421 421 self.__endUTCSecond - self.__timezone)
422 422 ))
423 423
424 424 print("[Reading] Starting process from %s to %s" % (datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
425 425 datetime.datetime.utcfromtimestamp(
426 426 endUTCSecond - self.__timezone)
427 427 ))
428 428 self.oldAverage = None
429 429 self.count = 0
430 430 self.executionTime = 0
431 431
432 432 def __reload(self):
433 433 # print
434 434 # print "%s not in range [%s, %s]" %(
435 435 # datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
436 436 # datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
437 437 # datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
438 438 # )
439 439 print("[Reading] reloading metadata ...")
440 440
441 441 try:
442 442 self.digitalReadObj.reload(complete_update=True)
443 443 except:
444 444 self.digitalReadObj = digital_rf.DigitalRFReader(self.path)
445 445
446 446 start_index, end_index = self.digitalReadObj.get_bounds(
447 447 self.__channelNameList[self.__channelList[0]])
448 448
449 449 if start_index > self.__startUTCSecond * self.__sample_rate:
450 450 self.__startUTCSecond = 1.0 * start_index / self.__sample_rate
451 451
452 452 if end_index > self.__endUTCSecond * self.__sample_rate:
453 453 self.__endUTCSecond = 1.0 * end_index / self.__sample_rate
454 454 print()
455 455 print("[Reading] New timerange found [%s, %s] " % (
456 456 datetime.datetime.utcfromtimestamp(
457 457 self.__startUTCSecond - self.__timezone),
458 458 datetime.datetime.utcfromtimestamp(
459 459 self.__endUTCSecond - self.__timezone)
460 460 ))
461 461
462 462 return True
463 463
464 464 return False
465 465
466 466 def timeit(self, toExecute):
467 467 t0 = time.time()
468 468 toExecute()
469 469 self.executionTime = time.time() - t0
470 470 if self.oldAverage is None:
471 471 self.oldAverage = self.executionTime
472 472 self.oldAverage = (self.executionTime + self.count *
473 473 self.oldAverage) / (self.count + 1.0)
474 474 self.count = self.count + 1.0
475 475 return
476 476
477 477 def __readNextBlock(self, seconds=30, volt_scale=1):
478 478 '''
479 479 '''
480 480
481 481 # Set the next data
482 482 self.__flagDiscontinuousBlock = False
483 483 self.__thisUnixSample += self.__samples_to_read
484 484
485 485 if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
486 486 print ("[Reading] There are no more data into selected time-range")
487 487 if self.__online:
488 488 sleep(3)
489 489 self.__reload()
490 490 else:
491 491 return False
492 492
493 493 if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
494 494 return False
495 495 self.__thisUnixSample -= self.__samples_to_read
496 496
497 497 indexChannel = 0
498 498
499 499 dataOk = False
500 500
501 501 for thisChannelName in self.__channelNameList: # TODO VARIOS CHANNELS?
502 502 for indexSubchannel in range(self.__num_subchannels):
503 503 try:
504 504 t0 = time()
505 505 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
506 506 self.__samples_to_read,
507 507 thisChannelName, sub_channel=indexSubchannel)
508 508 self.executionTime = time() - t0
509 509 if self.oldAverage is None:
510 510 self.oldAverage = self.executionTime
511 511 self.oldAverage = (
512 512 self.executionTime + self.count * self.oldAverage) / (self.count + 1.0)
513 513 self.count = self.count + 1.0
514 514
515 515 except IOError as e:
516 516 # read next profile
517 517 self.__flagDiscontinuousBlock = True
518 518 print("[Reading] %s" % datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e)
519 519 break
520 520
521 521 if result.shape[0] != self.__samples_to_read:
522 522 self.__flagDiscontinuousBlock = True
523 523 print("[Reading] %s: Too few samples were found, just %d/%d samples" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
524 524 result.shape[0],
525 525 self.__samples_to_read))
526 526 break
527 527
528 528 self.__data_buffer[indexChannel, :] = result * volt_scale
529 529 indexChannel+=1
530 530
531 531 dataOk = True
532 532
533 533 self.__utctime = self.__thisUnixSample / self.__sample_rate
534 534
535 535 if not dataOk:
536 536 return False
537 537
538 538 print("[Reading] %s: %d samples <> %f sec" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
539 539 self.__samples_to_read,
540 540 self.__timeInterval))
541 541
542 542 self.__bufferIndex = 0
543 543
544 544 return True
545 545
546 546 def __isBufferEmpty(self):
547 547 return self.__bufferIndex > self.__samples_to_read - self.__nSamples # 40960 - 40
548 548
549 549 def getData(self, seconds=30, nTries=5):
550 550 '''
551 551 This method gets the data from files and put the data into the dataOut object
552 552
553 553 In addition, increase el the buffer counter in one.
554 554
555 555 Return:
556 556 data : retorna un perfil de voltages (alturas * canales) copiados desde el
557 557 buffer. Si no hay mas archivos a leer retorna None.
558 558
559 559 Affected:
560 560 self.dataOut
561 561 self.profileIndex
562 562 self.flagDiscontinuousBlock
563 563 self.flagIsNewBlock
564 564 '''
565 565 #print("getdata")
566 566 err_counter = 0
567 567 self.dataOut.flagNoData = True
568 568
569 569 if self.__isBufferEmpty():
570 570 #print("hi")
571 571 self.__flagDiscontinuousBlock = False
572 572
573 573 while True:
574 574 #print ("q ha pasado")
575 575 if self.__readNextBlock():
576 576 break
577 577 if self.__thisUnixSample > self.__endUTCSecond * self.__sample_rate:
578 578 raise schainpy.admin.SchainError('Error')
579 579 return
580 580
581 581 if self.__flagDiscontinuousBlock:
582 582 raise schainpy.admin.SchainError('discontinuous block found')
583 583 return
584 584
585 585 if not self.__online:
586 586 raise schainpy.admin.SchainError('Online?')
587 587 return
588 588
589 589 err_counter += 1
590 590 if err_counter > nTries:
591 591 raise schainpy.admin.SchainError('Max retrys reach')
592 592 return
593 593
594 594 print('[Reading] waiting %d seconds to read a new block' % seconds)
595 595 sleep(seconds)
596 596
597 597 self.dataOut.data = self.__data_buffer[:, self.__bufferIndex:self.__bufferIndex + self.__nSamples]
598 598 self.dataOut.utctime = ( self.__thisUnixSample + self.__bufferIndex) / self.__sample_rate
599 599 self.dataOut.flagNoData = False
600 600 self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
601 601 self.dataOut.profileIndex = self.profileIndex
602 602
603 603 self.__bufferIndex += self.__nSamples
604 604 self.profileIndex += 1
605 605
606 606 if self.profileIndex == self.dataOut.nProfiles:
607 607 self.profileIndex = 0
608 608
609 609 return True
610 610
611 611 def printInfo(self):
612 612 '''
613 613 '''
614 614 if self.__printInfo == False:
615 615 return
616 616
617 617 # self.systemHeaderObj.printInfo()
618 618 # self.radarControllerHeaderObj.printInfo()
619 619
620 620 self.__printInfo = False
621 621
622 622 def printNumberOfBlock(self):
623 623 '''
624 624 '''
625 625 return
626 626 # print self.profileIndex
627 627
628 628 def run(self, **kwargs):
629 629 '''
630 630 This method will be called many times so here you should put all your code
631 631 '''
632 632
633 633 if not self.isConfig:
634 634 self.setup(**kwargs)
635 635 #self.i = self.i+1
636 636 self.getData(seconds=self.__delay)
637 637
638 638 return
639 639
640 640 @MPDecorator
641 641 class DigitalRFWriter(Operation):
642 642 '''
643 643 classdocs
644 644 '''
645 645
646 646 def __init__(self, **kwargs):
647 647 '''
648 648 Constructor
649 649 '''
650 650 Operation.__init__(self, **kwargs)
651 651 self.metadata_dict = {}
652 652 self.dataOut = None
653 653 self.dtype = None
654 654 self.oldAverage = 0
655 655
656 656 def setHeader(self):
657 657
658 658 self.metadata_dict['frequency'] = self.dataOut.frequency
659 659 self.metadata_dict['timezone'] = self.dataOut.timeZone
660 660 self.metadata_dict['dtype'] = pickle.dumps(self.dataOut.dtype)
661 661 self.metadata_dict['nProfiles'] = self.dataOut.nProfiles
662 662 self.metadata_dict['heightList'] = self.dataOut.heightList
663 663 self.metadata_dict['channelList'] = self.dataOut.channelList
664 664 self.metadata_dict['flagDecodeData'] = self.dataOut.flagDecodeData
665 665 self.metadata_dict['flagDeflipData'] = self.dataOut.flagDeflipData
666 666 self.metadata_dict['flagShiftFFT'] = self.dataOut.flagShiftFFT
667 667 self.metadata_dict['useLocalTime'] = self.dataOut.useLocalTime
668 668 self.metadata_dict['nCohInt'] = self.dataOut.nCohInt
669 669 self.metadata_dict['type'] = self.dataOut.type
670 670 self.metadata_dict['flagDataAsBlock']= getattr(
671 671 self.dataOut, 'flagDataAsBlock', None) # chequear
672 672
673 673 def setup(self, dataOut, path, frequency, fileCadence, dirCadence, metadataCadence, set=0, metadataFile='metadata', ext='.h5'):
674 674 '''
675 675 In this method we should set all initial parameters.
676 676 Input:
677 677 dataOut: Input data will also be outputa data
678 678 '''
679 679 self.setHeader()
680 680 self.__ippSeconds = dataOut.ippSeconds
681 681 self.__deltaH = dataOut.getDeltaH()
682 682 self.__sample_rate = 1e6 * 0.15 / self.__deltaH
683 683 self.__dtype = dataOut.dtype
684 684 if len(dataOut.dtype) == 2:
685 685 self.__dtype = dataOut.dtype[0]
686 686 self.__nSamples = dataOut.systemHeaderObj.nSamples
687 687 self.__nProfiles = dataOut.nProfiles
688 688
689 689 if self.dataOut.type != 'Voltage':
690 690 raise 'Digital RF cannot be used with this data type'
691 691 self.arr_data = numpy.ones((1, dataOut.nFFTPoints * len(
692 692 self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
693 693 else:
694 694 self.arr_data = numpy.ones((self.__nSamples, len(
695 695 self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
696 696
697 697 file_cadence_millisecs = 1000
698 698
699 699 sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator()
700 700 sample_rate_numerator = int(sample_rate_fraction.numerator)
701 701 sample_rate_denominator = int(sample_rate_fraction.denominator)
702 702 start_global_index = dataOut.utctime * self.__sample_rate
703 703
704 704 uuid = 'prueba'
705 705 compression_level = 0
706 706 checksum = False
707 707 is_complex = True
708 708 num_subchannels = len(dataOut.channelList)
709 709 is_continuous = True
710 710 marching_periods = False
711 711
712 712 self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, dirCadence,
713 713 fileCadence, start_global_index,
714 714 sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum,
715 715 is_complex, num_subchannels, is_continuous, marching_periods)
716 716 metadata_dir = os.path.join(path, 'metadata')
717 717 os.system('mkdir %s' % (metadata_dir))
718 718 self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, dirCadence, 1, # 236, file_cadence_millisecs / 1000
719 719 sample_rate_numerator, sample_rate_denominator,
720 720 metadataFile)
721 721 self.isConfig = True
722 722 self.currentSample = 0
723 723 self.oldAverage = 0
724 724 self.count = 0
725 725 return
726 726
727 727 def writeMetadata(self):
728 728 start_idx = self.__sample_rate * self.dataOut.utctime
729 729
730 730 self.metadata_dict['processingHeader'] = self.dataOut.processingHeaderObj.getAsDict(
731 731 )
732 732 self.metadata_dict['radarControllerHeader'] = self.dataOut.radarControllerHeaderObj.getAsDict(
733 733 )
734 734 self.metadata_dict['systemHeader'] = self.dataOut.systemHeaderObj.getAsDict(
735 735 )
736 736 self.digitalMetadataWriteObj.write(start_idx, self.metadata_dict)
737 737 return
738 738
739 739 def timeit(self, toExecute):
740 740 t0 = time()
741 741 toExecute()
742 742 self.executionTime = time() - t0
743 743 if self.oldAverage is None:
744 744 self.oldAverage = self.executionTime
745 745 self.oldAverage = (self.executionTime + self.count *
746 746 self.oldAverage) / (self.count + 1.0)
747 747 self.count = self.count + 1.0
748 748 return
749 749
750 750 def writeData(self):
751 751 if self.dataOut.type != 'Voltage':
752 752 raise 'Digital RF cannot be used with this data type'
753 753 for channel in self.dataOut.channelList:
754 754 for i in range(self.dataOut.nFFTPoints):
755 755 self.arr_data[1][channel * self.dataOut.nFFTPoints +
756 756 i]['r'] = self.dataOut.data[channel][i].real
757 757 self.arr_data[1][channel * self.dataOut.nFFTPoints +
758 758 i]['i'] = self.dataOut.data[channel][i].imag
759 759 else:
760 760 for i in range(self.dataOut.systemHeaderObj.nSamples):
761 761 for channel in self.dataOut.channelList:
762 762 self.arr_data[i][channel]['r'] = self.dataOut.data[channel][i].real
763 763 self.arr_data[i][channel]['i'] = self.dataOut.data[channel][i].imag
764 764
765 765 def f(): return self.digitalWriteObj.rf_write(self.arr_data)
766 766 self.timeit(f)
767 767
768 768 return
769 769
770 770 def run(self, dataOut, frequency=49.92e6, path=None, fileCadence=1000, dirCadence=36000, metadataCadence=1, **kwargs):
771 771 '''
772 772 This method will be called many times so here you should put all your code
773 773 Inputs:
774 774 dataOut: object with the data
775 775 '''
776 776 # print dataOut.__dict__
777 777 self.dataOut = dataOut
778 778 if not self.isConfig:
779 779 self.setup(dataOut, path, frequency, fileCadence,
780 780 dirCadence, metadataCadence, **kwargs)
781 781 self.writeMetadata()
782 782
783 783 self.writeData()
784 784
785 785 ## self.currentSample += 1
786 786 # if self.dataOut.flagDataAsBlock or self.currentSample == 1:
787 787 # self.writeMetadata()
788 788 ## if self.currentSample == self.__nProfiles: self.currentSample = 0
789 789
790 790 return dataOut# en la version 2.7 no aparece este return
791 791
792 792 def close(self):
793 793 print('[Writing] - Closing files ')
794 794 print('Average of writing to digital rf format is ', self.oldAverage * 1000)
795 795 try:
796 796 self.digitalWriteObj.close()
797 797 except:
798 798 pass
Show file before | Show file after | Hide comments
@@ -1,626 +1,648
1 1 import os
2 2 import time
3 3 import datetime
4 4
5 5 import numpy
6 6 import h5py
7 7
8 8 import schainpy.admin
9 9 from schainpy.model.data.jrodata import *
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 11 from schainpy.model.io.jroIO_base import *
12 12 from schainpy.utils import log
13 13
14 14
15 15 class HDFReader(Reader, ProcessingUnit):
16 16 """Processing unit to read HDF5 format files
17 17
18 18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 attributes.
20 attributes.
21 21 It is possible to read any HDF5 file by given the structure in the `description`
22 22 parameter, also you can add extra values to metadata with the parameter `extras`.
23 23
24 24 Parameters:
25 25 -----------
26 26 path : str
27 27 Path where files are located.
28 28 startDate : date
29 29 Start date of the files
30 30 endDate : list
31 31 End date of the files
32 32 startTime : time
33 33 Start time of the files
34 34 endTime : time
35 35 End time of the files
36 36 description : dict, optional
37 37 Dictionary with the description of the HDF5 file
38 38 extras : dict, optional
39 39 Dictionary with extra metadata to be be added to `dataOut`
40
40
41 41 Examples
42 42 --------
43
43
44 44 desc = {
45 45 'Data': {
46 46 'data_output': ['u', 'v', 'w'],
47 47 'utctime': 'timestamps',
48 48 } ,
49 49 'Metadata': {
50 50 'heightList': 'heights'
51 51 }
52 52 }
53 53
54 54 desc = {
55 55 'Data': {
56 56 'data_output': 'winds',
57 57 'utctime': 'timestamps'
58 58 },
59 59 'Metadata': {
60 60 'heightList': 'heights'
61 61 }
62 62 }
63 63
64 64 extras = {
65 65 'timeZone': 300
66 66 }
67
67
68 68 reader = project.addReadUnit(
69 69 name='HDFReader',
70 70 path='/path/to/files',
71 71 startDate='2019/01/01',
72 72 endDate='2019/01/31',
73 73 startTime='00:00:00',
74 74 endTime='23:59:59',
75 75 # description=json.dumps(desc),
76 76 # extras=json.dumps(extras),
77 77 )
78 78
79 79 """
80 80
81 81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82 82
83 83 def __init__(self):
84 84 ProcessingUnit.__init__(self)
85 85 self.dataOut = Parameters()
86 86 self.ext = ".hdf5"
87 87 self.optchar = "D"
88 88 self.meta = {}
89 89 self.data = {}
90 90 self.open_file = h5py.File
91 91 self.open_mode = 'r'
92 92 self.description = {}
93 93 self.extras = {}
94 94 self.filefmt = "*%Y%j***"
95 95 self.folderfmt = "*%Y%j"
96 96 self.utcoffset = 0
97 97
98 98 def setup(self, **kwargs):
99 99
100 100 self.set_kwargs(**kwargs)
101 101 if not self.ext.startswith('.'):
102 self.ext = '.{}'.format(self.ext)
102 self.ext = '.{}'.format(self.ext)
103 103
104 104 if self.online:
105 105 log.log("Searching files in online mode...", self.name)
106 106
107 107 for nTries in range(self.nTries):
108 108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
109 self.endDate, self.expLabel, self.ext, self.walk,
109 self.endDate, self.expLabel, self.ext, self.walk,
110 110 self.filefmt, self.folderfmt)
111 111 try:
112 112 fullpath = next(fullpath)
113 113 except:
114 114 fullpath = None
115
115
116 116 if fullpath:
117 117 break
118 118
119 119 log.warning(
120 120 'Waiting {} sec for a valid file in {}: try {} ...'.format(
121 self.delay, self.path, nTries + 1),
121 self.delay, self.path, nTries + 1),
122 122 self.name)
123 123 time.sleep(self.delay)
124 124
125 125 if not(fullpath):
126 126 raise schainpy.admin.SchainError(
127 'There isn\'t any valid file in {}'.format(self.path))
127 'There isn\'t any valid file in {}'.format(self.path))
128 128
129 129 pathname, filename = os.path.split(fullpath)
130 130 self.year = int(filename[1:5])
131 131 self.doy = int(filename[5:8])
132 self.set = int(filename[8:11]) - 1
132 self.set = int(filename[8:11]) - 1
133 133 else:
134 134 log.log("Searching files in {}".format(self.path), self.name)
135 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
135 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
136 136 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
137
137
138 138 self.setNextFile()
139 139
140 140 return
141 141
142 142 def readFirstHeader(self):
143 143 '''Read metadata and data'''
144 144
145 self.__readMetadata()
145 self.__readMetadata()
146 146 self.__readData()
147 147 self.__setBlockList()
148
148
149 149 if 'type' in self.meta:
150 150 self.dataOut = eval(self.meta['type'])()
151
151
152 152 for attr in self.meta:
153 153 setattr(self.dataOut, attr, self.meta[attr])
154
154
155 155 self.blockIndex = 0
156
156
157 157 return
158 158
159 159 def __setBlockList(self):
160 160 '''
161 161 Selects the data within the times defined
162 162
163 163 self.fp
164 164 self.startTime
165 165 self.endTime
166 166 self.blockList
167 167 self.blocksPerFile
168 168
169 169 '''
170 170
171 171 startTime = self.startTime
172 172 endTime = self.endTime
173 173 thisUtcTime = self.data['utctime'] + self.utcoffset
174 174 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
175 175 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
176 176
177 177 thisDate = thisDatetime.date()
178 178 thisTime = thisDatetime.time()
179 179
180 180 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
181 181 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
182 182
183 183 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
184 184
185 185 self.blockList = ind
186 186 self.blocksPerFile = len(ind)
187 187 return
188 188
189 189 def __readMetadata(self):
190 190 '''
191 191 Reads Metadata
192 192 '''
193 193
194 194 meta = {}
195 195
196 196 if self.description:
197 197 for key, value in self.description['Metadata'].items():
198 198 meta[key] = self.fp[value][()]
199 199 else:
200 200 grp = self.fp['Metadata']
201 201 for name in grp:
202 202 meta[name] = grp[name][()]
203 203
204 204 if self.extras:
205 205 for key, value in self.extras.items():
206 206 meta[key] = value
207 207 self.meta = meta
208 208
209 209 return
210 210
211 211 def __readData(self):
212 212
213 213 data = {}
214
214
215 215 if self.description:
216 216 for key, value in self.description['Data'].items():
217 217 if isinstance(value, str):
218 218 if isinstance(self.fp[value], h5py.Dataset):
219 219 data[key] = self.fp[value][()]
220 220 elif isinstance(self.fp[value], h5py.Group):
221 221 array = []
222 222 for ch in self.fp[value]:
223 223 array.append(self.fp[value][ch][()])
224 224 data[key] = numpy.array(array)
225 225 elif isinstance(value, list):
226 226 array = []
227 227 for ch in value:
228 228 array.append(self.fp[ch][()])
229 229 data[key] = numpy.array(array)
230 230 else:
231 231 grp = self.fp['Data']
232 232 for name in grp:
233 233 if isinstance(grp[name], h5py.Dataset):
234 234 array = grp[name][()]
235 235 elif isinstance(grp[name], h5py.Group):
236 236 array = []
237 237 for ch in grp[name]:
238 238 array.append(grp[name][ch][()])
239 239 array = numpy.array(array)
240 240 else:
241 241 log.warning('Unknown type: {}'.format(name))
242
242
243 243 if name in self.description:
244 244 key = self.description[name]
245 245 else:
246 246 key = name
247 247 data[key] = array
248 248
249 249 self.data = data
250 250 return
251
251
252 252 def getData(self):
253 253
254 254 for attr in self.data:
255 255 if self.data[attr].ndim == 1:
256 256 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
257 257 else:
258 258 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
259 259
260 260 self.dataOut.flagNoData = False
261 261 self.blockIndex += 1
262 262
263 263 log.log("Block No. {}/{} -> {}".format(
264 264 self.blockIndex,
265 265 self.blocksPerFile,
266 266 self.dataOut.datatime.ctime()), self.name)
267 267
268 268 return
269 269
270 270 def run(self, **kwargs):
271 271
272 272 if not(self.isConfig):
273 273 self.setup(**kwargs)
274 274 self.isConfig = True
275 275
276 276 if self.blockIndex == self.blocksPerFile:
277 277 self.setNextFile()
278 278
279 279 self.getData()
280 280
281 281 return
282 282
283 283 @MPDecorator
284 284 class HDFWriter(Operation):
285 285 """Operation to write HDF5 files.
286 286
287 287 The HDF5 file contains by default two groups Data and Metadata where
288 288 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
289 289 parameters, data attributes are normaly time dependent where the metadata
290 are not.
291 It is possible to customize the structure of the HDF5 file with the
290 are not.
291 It is possible to customize the structure of the HDF5 file with the
292 292 optional description parameter see the examples.
293 293
294 294 Parameters:
295 295 -----------
296 296 path : str
297 297 Path where files will be saved.
298 298 blocksPerFile : int
299 299 Number of blocks per file
300 300 metadataList : list
301 301 List of the dataOut attributes that will be saved as metadata
302 302 dataList : int
303 303 List of the dataOut attributes that will be saved as data
304 304 setType : bool
305 305 If True the name of the files corresponds to the timestamp of the data
306 306 description : dict, optional
307 307 Dictionary with the desired description of the HDF5 file
308
308
309 309 Examples
310 310 --------
311
311
312 312 desc = {
313 313 'data_output': {'winds': ['z', 'w', 'v']},
314 314 'utctime': 'timestamps',
315 315 'heightList': 'heights'
316 316 }
317 317 desc = {
318 318 'data_output': ['z', 'w', 'v'],
319 319 'utctime': 'timestamps',
320 320 'heightList': 'heights'
321 321 }
322 322 desc = {
323 323 'Data': {
324 324 'data_output': 'winds',
325 325 'utctime': 'timestamps'
326 326 },
327 327 'Metadata': {
328 328 'heightList': 'heights'
329 329 }
330 330 }
331
331
332 332 writer = proc_unit.addOperation(name='HDFWriter')
333 333 writer.addParameter(name='path', value='/path/to/file')
334 334 writer.addParameter(name='blocksPerFile', value='32')
335 335 writer.addParameter(name='metadataList', value='heightList,timeZone')
336 336 writer.addParameter(name='dataList',value='data_output,utctime')
337 337 # writer.addParameter(name='description',value=json.dumps(desc))
338 338
339 339 """
340 340
341 341 ext = ".hdf5"
342 342 optchar = "D"
343 343 filename = None
344 344 path = None
345 345 setFile = None
346 346 fp = None
347 347 firsttime = True
348 348 #Configurations
349 349 blocksPerFile = None
350 350 blockIndex = None
351 351 dataOut = None
352 352 #Data Arrays
353 353 dataList = None
354 354 metadataList = None
355 355 currentDay = None
356 356 lastTime = None
357 last_Azipos = None
358 last_Elepos = None
359 mode = None
360
361
357 362
358 363 def __init__(self):
359
364
360 365 Operation.__init__(self)
361 366 return
362 367
368 def generalFlag(self):
369 print("GENERALFLAG")
370 if self.mode== "weather":
371 if self.last_Azipos == None:
372 tmp = self.dataOut.azimuth
373 print("ang azimuth writer",tmp)
374 self.last_Azipos = tmp
375 flag = False
376 return flag
377 print("ang_azimuth writer",self.dataOut.azimuth)
378 result = self.dataOut.azimuth - self.last_Azipos
379 self.last_Azipos = self.dataOut.azimuth
380 if result<0:
381 flag = True
382 return flag
383
363 384 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
364 385 self.path = path
365 386 self.blocksPerFile = blocksPerFile
366 387 self.metadataList = metadataList
367 388 self.dataList = [s.strip() for s in dataList]
368 389 self.setType = setType
369 390 self.description = description
370 391
371 392 if self.metadataList is None:
372 393 self.metadataList = self.dataOut.metadata_list
373 394
374 395 tableList = []
375 396 dsList = []
376 397
377 398 for i in range(len(self.dataList)):
378 399 dsDict = {}
379 400 if hasattr(self.dataOut, self.dataList[i]):
380 401 dataAux = getattr(self.dataOut, self.dataList[i])
381 402 dsDict['variable'] = self.dataList[i]
382 403 else:
383 404 log.warning('Attribute {} not found in dataOut', self.name)
384 405 continue
385 406
386 407 if dataAux is None:
387 408 continue
388 409 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
389 410 dsDict['nDim'] = 0
390 411 else:
391 412 dsDict['nDim'] = len(dataAux.shape)
392 413 dsDict['shape'] = dataAux.shape
393 414 dsDict['dsNumber'] = dataAux.shape[0]
394 415 dsDict['dtype'] = dataAux.dtype
395
416
396 417 dsList.append(dsDict)
397 418
398 419 self.dsList = dsList
399 420 self.currentDay = self.dataOut.datatime.date()
400 421
401 422 def timeFlag(self):
402 423 currentTime = self.dataOut.utctime
403 424 timeTuple = time.localtime(currentTime)
404 425 dataDay = timeTuple.tm_yday
405 426
406 427 if self.lastTime is None:
407 428 self.lastTime = currentTime
408 429 self.currentDay = dataDay
409 430 return False
410
431
411 432 timeDiff = currentTime - self.lastTime
412 433
413 434 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
414 435 if dataDay != self.currentDay:
415 436 self.currentDay = dataDay
416 437 return True
417 438 elif timeDiff > 3*60*60:
418 439 self.lastTime = currentTime
419 440 return True
420 441 else:
421 442 self.lastTime = currentTime
422 443 return False
423 444
424 445 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
425 dataList=[], setType=None, description={}):
446 dataList=[], setType=None, description={},mode= None):
426 447
427 448 self.dataOut = dataOut
449 self.mode = mode
428 450 if not(self.isConfig):
429 self.setup(path=path, blocksPerFile=blocksPerFile,
451 self.setup(path=path, blocksPerFile=blocksPerFile,
430 452 metadataList=metadataList, dataList=dataList,
431 453 setType=setType, description=description)
432 454
433 455 self.isConfig = True
434 456 self.setNextFile()
435 457
436 458 self.putData()
437 459 return
438
460
439 461 def setNextFile(self):
440
462
441 463 ext = self.ext
442 464 path = self.path
443 465 setFile = self.setFile
444 466
445 467 timeTuple = time.localtime(self.dataOut.utctime)
446 468 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
447 469 fullpath = os.path.join(path, subfolder)
448 470
449 471 if os.path.exists(fullpath):
450 472 filesList = os.listdir(fullpath)
451 473 filesList = [k for k in filesList if k.startswith(self.optchar)]
452 474 if len( filesList ) > 0:
453 475 filesList = sorted(filesList, key=str.lower)
454 476 filen = filesList[-1]
455 477 # el filename debera tener el siguiente formato
456 478 # 0 1234 567 89A BCDE (hex)
457 479 # x YYYY DDD SSS .ext
458 480 if isNumber(filen[8:11]):
459 481 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
460 482 else:
461 483 setFile = -1
462 484 else:
463 485 setFile = -1 #inicializo mi contador de seteo
464 486 else:
465 487 os.makedirs(fullpath)
466 488 setFile = -1 #inicializo mi contador de seteo
467 489
468 490 if self.setType is None:
469 491 setFile += 1
470 492 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
471 493 timeTuple.tm_year,
472 494 timeTuple.tm_yday,
473 495 setFile,
474 496 ext )
475 497 else:
476 498 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
477 499 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
478 500 timeTuple.tm_year,
479 501 timeTuple.tm_yday,
480 502 setFile,
481 503 ext )
482 504
483 505 self.filename = os.path.join( path, subfolder, file )
484 506
485 507 #Setting HDF5 File
486 508 self.fp = h5py.File(self.filename, 'w')
487 509 #write metadata
488 510 self.writeMetadata(self.fp)
489 511 #Write data
490 512 self.writeData(self.fp)
491 513
492 514 def getLabel(self, name, x=None):
493 515
494 516 if x is None:
495 517 if 'Data' in self.description:
496 518 data = self.description['Data']
497 519 if 'Metadata' in self.description:
498 520 data.update(self.description['Metadata'])
499 521 else:
500 522 data = self.description
501 523 if name in data:
502 524 if isinstance(data[name], str):
503 525 return data[name]
504 526 elif isinstance(data[name], list):
505 527 return None
506 528 elif isinstance(data[name], dict):
507 529 for key, value in data[name].items():
508 530 return key
509 531 return name
510 532 else:
511 533 if 'Metadata' in self.description:
512 534 meta = self.description['Metadata']
513 535 else:
514 536 meta = self.description
515 537 if name in meta:
516 538 if isinstance(meta[name], list):
517 539 return meta[name][x]
518 540 elif isinstance(meta[name], dict):
519 541 for key, value in meta[name].items():
520 542 return value[x]
521 543 if 'cspc' in name:
522 544 return 'pair{:02d}'.format(x)
523 545 else:
524 546 return 'channel{:02d}'.format(x)
525
547
526 548 def writeMetadata(self, fp):
527 549
528 550 if self.description:
529 551 if 'Metadata' in self.description:
530 552 grp = fp.create_group('Metadata')
531 553 else:
532 554 grp = fp
533 555 else:
534 556 grp = fp.create_group('Metadata')
535 557
536 558 for i in range(len(self.metadataList)):
537 559 if not hasattr(self.dataOut, self.metadataList[i]):
538 560 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
539 561 continue
540 562 value = getattr(self.dataOut, self.metadataList[i])
541 563 if isinstance(value, bool):
542 564 if value is True:
543 565 value = 1
544 566 else:
545 567 value = 0
546 568 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
547 569 return
548 570
549 571 def writeData(self, fp):
550
572
551 573 if self.description:
552 574 if 'Data' in self.description:
553 575 grp = fp.create_group('Data')
554 576 else:
555 577 grp = fp
556 578 else:
557 579 grp = fp.create_group('Data')
558 580
559 581 dtsets = []
560 582 data = []
561
583
562 584 for dsInfo in self.dsList:
563 585 if dsInfo['nDim'] == 0:
564 586 ds = grp.create_dataset(
565 self.getLabel(dsInfo['variable']),
587 self.getLabel(dsInfo['variable']),
566 588 (self.blocksPerFile, ),
567 chunks=True,
589 chunks=True,
568 590 dtype=numpy.float64)
569 591 dtsets.append(ds)
570 592 data.append((dsInfo['variable'], -1))
571 593 else:
572 594 label = self.getLabel(dsInfo['variable'])
573 595 if label is not None:
574 596 sgrp = grp.create_group(label)
575 597 else:
576 598 sgrp = grp
577 599 for i in range(dsInfo['dsNumber']):
578 600 ds = sgrp.create_dataset(
579 self.getLabel(dsInfo['variable'], i),
601 self.getLabel(dsInfo['variable'], i),
580 602 (self.blocksPerFile, ) + dsInfo['shape'][1:],
581 603 chunks=True,
582 604 dtype=dsInfo['dtype'])
583 605 dtsets.append(ds)
584 606 data.append((dsInfo['variable'], i))
585 607 fp.flush()
586 608
587 609 log.log('Creating file: {}'.format(fp.filename), self.name)
588
610
589 611 self.ds = dtsets
590 612 self.data = data
591 613 self.firsttime = True
592 614 self.blockIndex = 0
593 615 return
594 616
595 617 def putData(self):
596
597 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
618 print("**************************PUT DATA***************************************************")
619 if (self.blockIndex == self.blocksPerFile) or self.timeFlag() or self.generalFlag():
598 620 self.closeFile()
599 621 self.setNextFile()
600 622
601 623 for i, ds in enumerate(self.ds):
602 624 attr, ch = self.data[i]
603 625 if ch == -1:
604 626 ds[self.blockIndex] = getattr(self.dataOut, attr)
605 627 else:
606 628 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
607 629
608 630 self.fp.flush()
609 631 self.blockIndex += 1
610 632 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
611 633
612 634 return
613 635
614 636 def closeFile(self):
615 637
616 638 if self.blockIndex != self.blocksPerFile:
617 639 for ds in self.ds:
618 640 ds.resize(self.blockIndex, axis=0)
619 641
620 642 if self.fp:
621 643 self.fp.flush()
622 644 self.fp.close()
623 645
624 646 def close(self):
625 647
626 648 self.closeFile()
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,464 +1,464
1 1 # scp.py
2 2 # Copyright (C) 2008 James Bardin <j.bardin@gmail.com>
3 3
4 4 """
5 5 Utilities for sending files over ssh using the scp1 protocol.
6 6 """
7 7
8 8 __version__ = '0.10.0'
9 9
10 10 import locale
11 11 import os
12 12 import re
13 13 from socket import timeout as SocketTimeout
14 14
15 15
16 16 # this is quote from the shlex module, added in py3.3
17 17 _find_unsafe = re.compile(br'[^\w@%+=:,./~-]').search
18 18
19 19
20 20 def _sh_quote(s):
21 21 """Return a shell-escaped version of the string `s`."""
22 22 if not s:
23 23 return b""
24 24 if _find_unsafe(s) is None:
25 25 return s
26 26
27 27 # use single quotes, and put single quotes into double quotes
28 28 # the string $'b is then quoted as '$'"'"'b'
29 29 return b"'" + s.replace(b"'", b"'\"'\"'") + b"'"
30 30
31 31
32 32 # Unicode conversion functions; assume UTF-8
33 33
34 34 def asbytes(s):
35 35 """Turns unicode into bytes, if needed.
36 36 Assumes UTF-8.
37 37 """
38 38 if isinstance(s, bytes):
39 39 return s
40 40 else:
41 41 return s.encode('utf-8')
42 42
43 43
44 44 def asunicode(s):
45 45 """Turns bytes into unicode, if needed.
46 46 Uses UTF-8.
47 47 """
48 48 if isinstance(s, bytes):
49 49 return s.decode('utf-8', 'replace')
50 50 else:
51 51 return s
52 52
53 53
54 54 # os.path.sep is unicode on Python 3, no matter the platform
55 55 bytes_sep = asbytes(os.path.sep)
56 56
57 57
58 58 # Unicode conversion function for Windows
59 59 # Used to convert local paths if the local machine is Windows
60 60
61 61 def asunicode_win(s):
62 62 """Turns bytes into unicode, if needed.
63 63 """
64 64 if isinstance(s, bytes):
65 65 return s.decode(locale.getpreferredencoding())
66 66 else:
67 67 return s
68 68
69 69
70 70 class SCPClient(object):
71 71 """
72 72 An scp1 implementation, compatible with openssh scp.
73 73 Raises SCPException for all transport related errors. Local filesystem
74 74 and OS errors pass through.
75 75 Main public methods are .put and .get
76 76 The get method is controlled by the remote scp instance, and behaves
77 77 accordingly. This means that symlinks are resolved, and the transfer is
78 78 halted after too many levels of symlinks are detected.
79 79 The put method uses os.walk for recursion, and sends files accordingly.
80 80 Since scp doesn't support symlinks, we send file symlinks as the file
81 81 (matching scp behaviour), but we make no attempt at symlinked directories.
82 82 """
83 83 def __init__(self, transport, buff_size=16384, socket_timeout=5.0,
84 84 progress=None, sanitize=_sh_quote):
85 85 """
86 86 Create an scp1 client.
87 87 @param transport: an existing paramiko L{Transport}
88 88 @type transport: L{Transport}
89 89 @param buff_size: size of the scp send buffer.
90 90 @type buff_size: int
91 91 @param socket_timeout: channel socket timeout in seconds
92 92 @type socket_timeout: float
93 93 @param progress: callback - called with (filename, size, sent) during
94 94 transfers
95 95 @param sanitize: function - called with filename, should return
96 96 safe or escaped string. Uses _sh_quote by default.
97 97 @type progress: function(string, int, int)
98 98 """
99 99 self.transport = transport
100 100 self.buff_size = buff_size
101 101 self.socket_timeout = socket_timeout
102 102 self.channel = None
103 103 self.preserve_times = False
104 104 self._progress = progress
105 105 self._recv_dir = b''
106 106 self._rename = False
107 107 self._utime = None
108 108 self.sanitize = sanitize
109 109 self._dirtimes = {}
110 110
111 111 def __enter__(self):
112 112 self.channel = self._open()
113 113 return self
114 114
115 115 def __exit__(self, type, value, traceback):
116 116 self.close()
117 117
118 118 def put(self, files, remote_path=b'.',
119 119 recursive=False, preserve_times=False):
120 120 """
121 121 Transfer files to remote host.
122 122 @param files: A single path, or a list of paths to be transfered.
123 123 recursive must be True to transfer directories.
124 124 @type files: string OR list of strings
125 125 @param remote_path: path in which to receive the files on the remote
126 126 host. defaults to '.'
127 127 @type remote_path: str
128 128 @param recursive: transfer files and directories recursively
129 129 @type recursive: bool
130 130 @param preserve_times: preserve mtime and atime of transfered files
131 131 and directories.
132 132 @type preserve_times: bool
133 133 """
134 134 self.preserve_times = preserve_times
135 135 self.channel = self._open()
136 136 self._pushed = 0
137 137 self.channel.settimeout(self.socket_timeout)
138 138 scp_command = (b'scp -t ', b'scp -r -t ')[recursive]
139 139 self.channel.exec_command(scp_command +
140 140 self.sanitize(asbytes(remote_path)))
141 141 self._recv_confirm()
142 142
143 143 if not isinstance(files, (list, tuple)):
144 144 files = [files]
145 145
146 146 if recursive:
147 147 self._send_recursive(files)
148 148 else:
149 149 self._send_files(files)
150 150
151 151 self.close()
152 152
153 153 def get(self, remote_path, local_path='',
154 154 recursive=False, preserve_times=False):
155 155 """
156 156 Transfer files from remote host to localhost
157 157 @param remote_path: path to retreive from remote host. since this is
158 158 evaluated by scp on the remote host, shell wildcards and
159 159 environment variables may be used.
160 160 @type remote_path: str
161 161 @param local_path: path in which to receive files locally
162 162 @type local_path: str
163 163 @param recursive: transfer files and directories recursively
164 164 @type recursive: bool
165 165 @param preserve_times: preserve mtime and atime of transfered files
166 166 and directories.
167 167 @type preserve_times: bool
168 168 """
169 169 if not isinstance(remote_path, (list, tuple)):
170 170 remote_path = [remote_path]
171 171 remote_path = [self.sanitize(asbytes(r)) for r in remote_path]
172 172 self._recv_dir = local_path or os.getcwd()
173 173 self._rename = (len(remote_path) == 1 and
174 174 not os.path.isdir(os.path.abspath(local_path)))
175 175 if len(remote_path) > 1:
176 176 if not os.path.exists(self._recv_dir):
177 177 raise SCPException("Local path '%s' does not exist" %
178 178 asunicode(self._recv_dir))
179 179 elif not os.path.isdir(self._recv_dir):
180 180 raise SCPException("Local path '%s' is not a directory" %
181 181 asunicode(self._recv_dir))
182 182 rcsv = (b'', b' -r')[recursive]
183 183 prsv = (b'', b' -p')[preserve_times]
184 184 self.channel = self._open()
185 185 self._pushed = 0
186 186 self.channel.settimeout(self.socket_timeout)
187 187 self.channel.exec_command(b"scp" +
188 188 rcsv +
189 189 prsv +
190 190 b" -f " +
191 191 b' '.join(remote_path))
192 192 self._recv_all()
193 193 self.close()
194 194
195 195 def _open(self):
196 196 """open a scp channel"""
197 197 if self.channel is None:
198 198 self.channel = self.transport.open_session()
199 199
200 200 return self.channel
201 201
202 202 def close(self):
203 203 """close scp channel"""
204 204 if self.channel is not None:
205 205 self.channel.close()
206 206 self.channel = None
207 207
208 208 def _read_stats(self, name):
209 209 """return just the file stats needed for scp"""
210 210 if os.name == 'nt':
211 211 name = asunicode(name)
212 212 stats = os.stat(name)
213 213 mode = oct(stats.st_mode)[-4:]
214 214 size = stats.st_size
215 215 atime = int(stats.st_atime)
216 216 mtime = int(stats.st_mtime)
217 217 return (mode, size, mtime, atime)
218 218
219 219 def _send_files(self, files):
220 220 for name in files:
221 221 basename = asbytes(os.path.basename(name))
222 222 (mode, size, mtime, atime) = self._read_stats(name)
223 223 if self.preserve_times:
224 224 self._send_time(mtime, atime)
225 225 file_hdl = open(name, 'rb')
226 226
227 227 # The protocol can't handle \n in the filename.
228 228 # Quote them as the control sequence \^J for now,
229 229 # which is how openssh handles it.
230 230 self.channel.sendall(("C%s %d " % (mode, size)).encode('ascii') +
231 231 basename.replace(b'\n', b'\\^J') + b"\n")
232 232 self._recv_confirm()
233 233 file_pos = 0
234 234 if self._progress:
235 235 if size == 0:
236 236 # avoid divide-by-zero
237 237 self._progress(basename, 1, 1)
238 238 else:
239 239 self._progress(basename, size, 0)
240 240 buff_size = self.buff_size
241 241 chan = self.channel
242 242 while file_pos < size:
243 243 chan.sendall(file_hdl.read(buff_size))
244 244 file_pos = file_hdl.tell()
245 245 if self._progress:
246 246 self._progress(basename, size, file_pos)
247 247 chan.sendall('\x00')
248 248 file_hdl.close()
249 249 self._recv_confirm()
250 250
251 251 def _chdir(self, from_dir, to_dir):
252 252 # Pop until we're one level up from our next push.
253 253 # Push *once* into to_dir.
254 254 # This is dependent on the depth-first traversal from os.walk
255 255
256 256 # add path.sep to each when checking the prefix, so we can use
257 257 # path.dirname after
258 258 common = os.path.commonprefix([from_dir + bytes_sep,
259 259 to_dir + bytes_sep])
260 260 # now take the dirname, since commonprefix is character based,
261 261 # and we either have a seperator, or a partial name
262 262 common = os.path.dirname(common)
263 263 cur_dir = from_dir.rstrip(bytes_sep)
264 264 while cur_dir != common:
265 265 cur_dir = os.path.split(cur_dir)[0]
266 266 self._send_popd()
267 267 # now we're in our common base directory, so on
268 268 self._send_pushd(to_dir)
269 269
270 270 def _send_recursive(self, files):
271 271 for base in files:
272 272 if not os.path.isdir(base):
273 273 # filename mixed into the bunch
274 274 self._send_files([base])
275 275 continue
276 276 last_dir = asbytes(base)
277 277 for root, dirs, fls in os.walk(base):
278 278 self._chdir(last_dir, asbytes(root))
279 279 self._send_files([os.path.join(root, f) for f in fls])
280 280 last_dir = asbytes(root)
281 281 # back out of the directory
282 282 while self._pushed > 0:
283 283 self._send_popd()
284 284
285 285 def _send_pushd(self, directory):
286 286 (mode, size, mtime, atime) = self._read_stats(directory)
287 287 basename = asbytes(os.path.basename(directory))
288 288 if self.preserve_times:
289 289 self._send_time(mtime, atime)
290 290 self.channel.sendall(('D%s 0 ' % mode).encode('ascii') +
291 291 basename.replace(b'\n', b'\\^J') + b'\n')
292 292 self._recv_confirm()
293 293 self._pushed += 1
294 294
295 295 def _send_popd(self):
296 296 self.channel.sendall('E\n')
297 297 self._recv_confirm()
298 298 self._pushed -= 1
299 299
300 300 def _send_time(self, mtime, atime):
301 301 self.channel.sendall(('T%d 0 %d 0\n' % (mtime, atime)).encode('ascii'))
302 302 self._recv_confirm()
303 303
304 304 def _recv_confirm(self):
305 305 # read scp response
306 306 msg = b''
307 307 try:
308 308 msg = self.channel.recv(512)
309 309 except SocketTimeout:
310 310 raise SCPException('Timout waiting for scp response')
311 311 # slice off the first byte, so this compare will work in py2 and py3
312 312 if msg and msg[0:1] == b'\x00':
313 313 return
314 314 elif msg and msg[0:1] == b'\x01':
315 315 raise SCPException(asunicode(msg[1:]))
316 316 elif self.channel.recv_stderr_ready():
317 317 msg = self.channel.recv_stderr(512)
318 318 raise SCPException(asunicode(msg))
319 319 elif not msg:
320 320 raise SCPException('No response from server')
321 321 else:
322 322 raise SCPException('Invalid response from server', msg)
323 323
324 324 def _recv_all(self):
325 325 # loop over scp commands, and receive as necessary
326 326 command = {b'C': self._recv_file,
327 327 b'T': self._set_time,
328 328 b'D': self._recv_pushd,
329 329 b'E': self._recv_popd}
330 330 while not self.channel.closed:
331 331 # wait for command as long as we're open
332 332 self.channel.sendall('\x00')
333 333 msg = self.channel.recv(1024)
334 334 if not msg: # chan closed while recving
335 335 break
336 336 assert msg[-1:] == b'\n'
337 337 msg = msg[:-1]
338 338 code = msg[0:1]
339 339 try:
340 340 command[code](msg[1:])
341 341 except KeyError:
342 342 raise SCPException(asunicode(msg[1:]))
343 343 # directory times can't be set until we're done writing files
344 344 self._set_dirtimes()
345 345
346 346 def _set_time(self, cmd):
347 347 try:
348 348 times = cmd.split(b' ')
349 349 mtime = int(times[0])
350 350 atime = int(times[2]) or mtime
351 351 except:
352 352 self.channel.send(b'\x01')
353 353 raise SCPException('Bad time format')
354 354 # save for later
355 355 self._utime = (atime, mtime)
356 356
357 357 def _recv_file(self, cmd):
358 358 chan = self.channel
359 359 parts = cmd.strip().split(b' ', 2)
360 360
361 361 try:
362 362 mode = int(parts[0], 8)
363 363 size = int(parts[1])
364 364 if self._rename:
365 365 path = self._recv_dir
366 366 self._rename = False
367 367 elif os.name == 'nt':
368 368 path = os.path.join(asunicode_win(self._recv_dir),
369 369 parts[2].decode('utf-8'))
370 370 else:
371 371 path = os.path.join(asbytes(self._recv_dir),
372 372 parts[2])
373 373 except:
374 374 chan.send('\x01')
375 375 chan.close()
376 376 raise SCPException('Bad file format')
377 377
378 378 try:
379 379 file_hdl = open(path, 'wb')
380 380 except IOError as e:
381 381 chan.send(b'\x01' + str(e).encode('utf-8'))
382 382 chan.close()
383 383 raise
384 384
385 385 if self._progress:
386 386 if size == 0:
387 387 # avoid divide-by-zero
388 388 self._progress(path, 1, 1)
389 389 else:
390 390 self._progress(path, size, 0)
391 391 buff_size = self.buff_size
392 392 pos = 0
393 393 chan.send(b'\x00')
394 394 try:
395 395 while pos < size:
396 396 # we have to make sure we don't read the final byte
397 397 if size - pos <= buff_size:
398 398 buff_size = size - pos
399 399 file_hdl.write(chan.recv(buff_size))
400 400 pos = file_hdl.tell()
401 401 if self._progress:
402 402 self._progress(path, size, pos)
403 403
404 404 msg = chan.recv(512)
405 405 if msg and msg[0:1] != b'\x00':
406 406 raise SCPException(asunicode(msg[1:]))
407 407 except SocketTimeout:
408 408 chan.close()
409 409 raise SCPException('Error receiving, socket.timeout')
410 410
411 411 file_hdl.truncate()
412 412 try:
413 413 os.utime(path, self._utime)
414 414 self._utime = None
415 415 os.chmod(path, mode)
416 416 # should we notify the other end?
417 417 finally:
418 418 file_hdl.close()
419 419 # '\x00' confirmation sent in _recv_all
420 420
421 421 def _recv_pushd(self, cmd):
422 422 parts = cmd.split(b' ', 2)
423 423 try:
424 424 mode = int(parts[0], 8)
425 425 if self._rename:
426 426 path = self._recv_dir
427 427 self._rename = False
428 428 elif os.name == 'nt':
429 429 path = os.path.join(asunicode_win(self._recv_dir),
430 430 parts[2].decode('utf-8'))
431 431 else:
432 432 path = os.path.join(asbytes(self._recv_dir),
433 433 parts[2])
434 434 except:
435 435 self.channel.send(b'\x01')
436 436 raise SCPException('Bad directory format')
437 437 try:
438 438 if not os.path.exists(path):
439 439 os.mkdir(path, mode)
440 440 elif os.path.isdir(path):
441 441 os.chmod(path, mode)
442 442 else:
443 443 raise SCPException('%s: Not a directory' % path)
444 444 self._dirtimes[path] = (self._utime)
445 445 self._utime = None
446 446 self._recv_dir = path
447 447 except (OSError, SCPException) as e:
448 448 self.channel.send(b'\x01' + asbytes(str(e)))
449 449 raise
450 450
451 451 def _recv_popd(self, *cmd):
452 452 self._recv_dir = os.path.split(self._recv_dir)[0]
453 453
454 454 def _set_dirtimes(self):
455 455 try:
456 456 for d in self._dirtimes:
457 457 os.utime(d, self._dirtimes[d])
458 458 finally:
459 459 self._dirtimes = {}
460 460
461 461
462 462 class SCPException(Exception):
463 463 """SCP exception class"""
464 pass No newline at end of file
464 pass
Show file before | Show file after | Hide comments
@@ -1,216 +1,282
1 1 # Ing. AVP
2 2 # 06/10/2021
3 3 # ARCHIVO DE LECTURA
4 4 import os, sys
5 5 import datetime
6 6 import time
7 7 from schainpy.controller import Project
8 8 #### NOTA###########################################
9 9 # INPUT :
10 10 # VELOCIDAD PARAMETRO : V = 2Β°/seg
11 11 # MODO PULSE PAIR O MOMENTOS: 0 : Pulse Pair ,1 : Momentos
12 12 ######################################################
13 13 ##### PROCESAMIENTO ##################################
14 14 ##### OJO TENER EN CUENTA EL n= para el Pulse Pair ##
15 15 ##### O EL n= nFFTPoints ###
16 16 ######################################################
17 17 ######## BUSCAMOS EL numero de IPP equivalente 1Β°#####
18 18 ######## Sea V la velocidad del Pedestal en Β°/seg#####
19 19 ######## 1Β° sera Recorrido en un tiempo de 1/V ######
20 20 ######## IPP del Radar 400 useg --> 60 Km ############
21 21 ######## n = 1/(V(Β°/seg)*IPP(Km)) , NUMERO DE IPP ##
22 22 ######## n = 1/(V*IPP) #############################
23 23 ######## VELOCIDAD DEL PEDESTAL ######################
24 24 print("SETUP- RADAR METEOROLOGICO")
25 V = 10
25 V = 6
26 26 mode = 1
27 #path = '/DATA_RM/23/6v'
28 ####path = '/DATA_RM/TEST_INTEGRACION_2M'
29 #path = '/DATA_RM/TEST_19OCTUBRE/10MHZ'
30 path = '/DATA_RM/WR_20_OCT'
27 #--------------------------PATH -----------------------------
28 # path = '/DATA_RM/23/6v'
29 # path = '/DATA_RM/TEST_INTEGRACION_2M'
30 # path = '/DATA_RM/TEST_19OCTUBRE/10MHZ'
31 # path = '/DATA_RM/WR_20_OCT'
31 32 #### path_ped='/DATA_RM/TEST_PEDESTAL/P20211012-082745'
32 33 ####path_ped='/DATA_RM/TEST_PEDESTAL/P20211019-192244'
33 figpath_pp = "/home/soporte/Pictures/TEST_PP"
34 figpath_spec = "/home/soporte/Pictures/TEST_MOM"
34 #path = '/DATA_RM/WR_POT_09_1'
35 #path = '/DATA_RM/WR_POT_09_2'
36 #path ="/DATA_RM/10"
37 #path ="/DATA_RM/11"
38 #path ="/DATA_RM/DRONE/2MHZ_6V"
39 path ="/DATA_RM/DRONE/2MHZ_10V_ELEVACION"
40 path ="/DATA_RM/DRONE/2MHZ_6V_AZIMUTH"
41 #-------------------------PATH-PLOTEO------------------------------------
42 #figpath_pp = "/home/soporte/Pictures/TEST_PP"
43 #figpath_spec = "/home/soporte/Pictures/TEST_MOM_15"
44 #figpath_spec = "/home/soporte/Pictures/11_"
45 figpath_spec = "/home/soporte/Pictures/23NOV"
46 figpath_spec = "/home/soporte/Pictures/23NOV_TEST_AZI"
47 figpath_pp = "/home/soporte/Pictures/TEST_POT"
48 figpath_pp = "/home/soporte/Pictures/TEST_POT_10_"
49
50
51 #--------------------------OPCIONES-----------------------------------
35 52 plot = 0
36 integration = 1
53 integration = 0
37 54 save = 0
55 plot_spec = 1
56 #-------------------------------------------------------------------------
38 57 if save == 1:
39 58 if mode==0:
40 59 path_save = '/DATA_RM/TEST_HDF5_PP_23/6v'
41 60 path_save = '/DATA_RM/TEST_HDF5_PP'
42 61 path_save = '/DATA_RM/TEST_HDF5_PP_100'
43 62 else:
44 63 path_save = '/DATA_RM/TEST_HDF5_SPEC_23_V2/6v'
45 64
46 65 print("* PATH data ADQ :", path)
47 66 print("* Velocidad Pedestal :",V,"Β°/seg")
48 67 ############################ NRO Perfiles PROCESAMIENTO ###################
49 68 V=V
50 69 IPP=400*1e-6
51 70 n= int(1/(V*IPP))
52 71 print("* n - NRO Perfiles Proc:", n )
53 72 ################################## MODE ###################################
54 73 print("* Modo de Operacion :",mode)
55 74 if mode ==0:
56 75 print("* Met. Seleccionado : Pulse Pair")
57 76 else:
58 77 print("* Met. Momentos : Momentos")
59 78
60 79 ################################## MODE ###################################
61 80 print("* Grabado de datos :",save)
62 81 if save ==1:
63 82 if mode==0:
64 83 ope= "Pulse Pair"
65 84 else:
66 85 ope= "Momentos"
67 86 print("* Path-Save Data -", ope , path_save)
68 87
69 88 print("* Integracion de datos :",integration)
70 89
71 time.sleep(15)
90 time.sleep(3)
72 91 #remotefolder = "/home/wmaster/graficos"
73 92 #######################################################################
74 93 ################# RANGO DE PLOTEO######################################
75 dBmin = '1'
76 dBmax = '65'
77 xmin = '13.2'
78 xmax = '13.5'
79 ymin = '0'
80 ymax = '60'
94 dBmin = '8'
95 dBmax = '35'
96 xmin = '12.1'#17.1,17.5
97 xmax = '12.2'#17.2,17.8
98 ymin = '0'#### PONER A 0
99 ymax = '8'#### PONER A 8
81 100 #######################################################################
82 101 ########################FECHA##########################################
83 102 str = datetime.date.today()
84 103 today = str.strftime("%Y/%m/%d")
85 104 str2 = str - datetime.timedelta(days=1)
86 105 yesterday = str2.strftime("%Y/%m/%d")
87 106 #######################################################################
88 107 ########################SIGNAL CHAIN ##################################
89 108 #######################################################################
90 109 desc = "USRP_test"
91 110 filename = "USRP_processing.xml"
92 111 controllerObj = Project()
93 112 controllerObj.setup(id = '191', name='Test_USRP', description=desc)
94 113 #######################################################################
95 114 ######################## UNIDAD DE LECTURA#############################
96 115 #######################################################################
97 116 readUnitConfObj = controllerObj.addReadUnit(datatype='DigitalRFReader',
98 117 path=path,
99 startDate="2021/01/01",#today,
100 endDate="2021/12/30",#today,
101 startTime='00:00:00',
102 endTime='23:59:59',
118 startDate="2021/11/23",#today,
119 endDate="2021/12/23",#today,
120 startTime='15:00:00',#'17:39:25',
121 endTime='16:00:59',#23:59:59',
103 122 delay=0,
104 123 #set=0,
105 online=0,
124 online=1,
106 125 walk=1,
107 126 ippKm = 60)
108 127
109 128 opObj11 = readUnitConfObj.addOperation(name='printInfo')
110 129
111 130 procUnitConfObjA = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
112 131
132 opObj11 = procUnitConfObjA.addOperation(name='setH0')
133 opObj11.addParameter(name='h0', value='-3.3', format='float')
134 #-2.8 5MHZ
135 #-3.3 2MHZ
136
137 ###opObj11 = procUnitConfObjA.addOperation(name='selectChannels')
138 ###opObj11.addParameter(name='channelList', value='[1]',format='intList')
139
140
141 opObj11 = procUnitConfObjA.addOperation(name='selectHeights')
142 opObj11.addParameter(name='minIndex', value='1', format='int')
143 ###opObj11.addParameter(name='maxIndex', value='10000', format='int')
144 opObj11.addParameter(name='maxIndex', value='160', format='int')
145 ## 400 PARA 5MHZ 12KM 2000
146 ## 160 αΉ”ARA 2MHZ 12KM 800
147
113 148 if mode ==0:
114 149 ####################### METODO PULSE PAIR ######################################################################
115 150 opObj11 = procUnitConfObjA.addOperation(name='PulsePair', optype='other')
116 151 opObj11.addParameter(name='n', value=int(n), format='int')#10 VOY A USAR 250 DADO QUE LA VELOCIDAD ES 10 GRADOS
117 152 #opObj11.addParameter(name='removeDC', value=1, format='int')
118 153 ####################### METODO Parametros ######################################################################
119 154 procUnitConfObjB= controllerObj.addProcUnit(datatype='ParametersProc',inputId=procUnitConfObjA.getId())
120 155 if plot==1:
121 156 opObj11 = procUnitConfObjB.addOperation(name='GenericRTIPlot',optype='external')
122 157 opObj11.addParameter(name='attr_data', value='dataPP_POWER')
123 158 opObj11.addParameter(name='colormap', value='jet')
124 159 opObj11.addParameter(name='xmin', value=xmin)
125 160 opObj11.addParameter(name='xmax', value=xmax)
161 #opObj11.addParameter(name='ymin', value=ymin)
162 #opObj11.addParameter(name='ymax', value=ymax)
126 163 opObj11.addParameter(name='zmin', value=dBmin)
127 164 opObj11.addParameter(name='zmax', value=dBmax)
128 165 opObj11.addParameter(name='save', value=figpath_pp)
129 166 opObj11.addParameter(name='showprofile', value=0)
130 167 opObj11.addParameter(name='save_period', value=10)
131 168
132 169 ####################### METODO ESCRITURA #######################################################################
133 170 if save==1:
134 171 opObj10 = procUnitConfObjB.addOperation(name='HDFWriter')
135 172 opObj10.addParameter(name='path',value=path_save)
136 173 #opObj10.addParameter(name='mode',value=0)
137 174 opObj10.addParameter(name='blocksPerFile',value='100',format='int')
138 175 opObj10.addParameter(name='metadataList',value='utctimeInit,timeZone,paramInterval,profileIndex,channelList,heightList,flagDataAsBlock',format='list')
139 176 opObj10.addParameter(name='dataList',value='dataPP_POWER,dataPP_DOP,utctime',format='list')#,format='list'
140 177 if integration==1:
141 178 V=10
142 179 blocksPerfile=360
143 180 print("* Velocidad del Pedestal:",V)
144 181 tmp_blocksPerfile = 100
145 182 f_a_p= int(tmp_blocksPerfile/V)
146 183
147 184 opObj11 = procUnitConfObjB.addOperation(name='PedestalInformation')
148 185 opObj11.addParameter(name='path_ped', value=path_ped)
149 186 #opObj11.addParameter(name='path_adq', value=path_adq)
150 187 opObj11.addParameter(name='t_Interval_p', value='0.01', format='float')
151 188 opObj11.addParameter(name='blocksPerfile', value=blocksPerfile, format='int')
152 189 opObj11.addParameter(name='n_Muestras_p', value='100', format='float')
153 190 opObj11.addParameter(name='f_a_p', value=f_a_p, format='int')
154 191 opObj11.addParameter(name='online', value='0', format='int')
155 192
156 193 opObj11 = procUnitConfObjB.addOperation(name='Block360')
157 194 opObj11.addParameter(name='n', value='10', format='int')
158 195 opObj11.addParameter(name='mode', value=mode, format='int')
159 196
160 197 # este bloque funciona bien con divisores de 360 no olvidar 0 10 20 30 40 60 90 120 180
161 198
162 199 opObj11= procUnitConfObjB.addOperation(name='WeatherPlot',optype='other')
163 200
164 201
165 202 else:
166 203 ####################### METODO SPECTROS ######################################################################
167 204 procUnitConfObjB = controllerObj.addProcUnit(datatype='SpectraProc', inputId=procUnitConfObjA.getId())
168 205 procUnitConfObjB.addParameter(name='nFFTPoints', value=n, format='int')
169 206 procUnitConfObjB.addParameter(name='nProfiles' , value=n, format='int')
207 if plot_spec==1:
208 '''
209 opObj11 = procUnitConfObjB.addOperation(name='SpectraPlot', optype='external')
210 #.addParameter(name='id', value='2', format='int')
211 opObj11.addParameter(name='wintitle', value='SpectraPlot', format='str')
212 opObj11.addParameter(name='channelList', value='1', format='intlist')
213 #opObj11.addParameter(name='xmin', value=xmin)
214 #opObj11.addParameter(name='xmax', value=xmax)
215 opObj11.addParameter(name='ymin', value=ymin)
216 opObj11.addParameter(name='ymax', value=ymax)
217 opObj11.addParameter(name='zmin', value=dBmin, format='int')
218 opObj11.addParameter(name='zmax', value=dBmax, format='int')
219 opObj11.addParameter(name='save', value=figpath_spec)
220 opObj11.addParameter(name='showprofile', value=0)
221 opObj11.addParameter(name='save_period', value=1)
222 '''
223 opObj11 = procUnitConfObjB.addOperation(name='RTIPlot', optype='external')
224 #.addParameter(name='id', value='2', format='int')
225 opObj11.addParameter(name='wintitle', value='RTIPlot', format='str')
226 opObj11.addParameter(name='channelList', value='1', format='intlist')
227 opObj11.addParameter(name='xmin', value=xmin)
228 opObj11.addParameter(name='xmax', value=xmax)
229 opObj11.addParameter(name='ymin', value=ymin)
230 opObj11.addParameter(name='ymax', value=ymax)
231 opObj11.addParameter(name='zmin', value=dBmin, format='int')
232 opObj11.addParameter(name='zmax', value=dBmax, format='int')
233 opObj11.addParameter(name='save', value=figpath_spec)
234 opObj11.addParameter(name='showprofile', value=0)
235 opObj11.addParameter(name='save_period', value=1)
170 236
171 237 procUnitConfObjC = controllerObj.addProcUnit(datatype='ParametersProc',inputId=procUnitConfObjB.getId())
172 238 procUnitConfObjC.addOperation(name='SpectralMoments')
173 239 if plot==1:
174 dBmin = '1'
175 dBmax = '65'
176 240 opObj11 = procUnitConfObjC.addOperation(name='PowerPlot',optype='external')
177 241 opObj11.addParameter(name='xmin', value=xmin)
178 242 opObj11.addParameter(name='xmax', value=xmax)
243 opObj11.addParameter(name='ymin', value=ymin)
244 opObj11.addParameter(name='ymax', value=ymax)
179 245 opObj11.addParameter(name='zmin', value=dBmin)
180 246 opObj11.addParameter(name='zmax', value=dBmax)
181 247 opObj11.addParameter(name='save', value=figpath_spec)
182 248 opObj11.addParameter(name='showprofile', value=0)
183 opObj11.addParameter(name='save_period', value=10)
249 opObj11.addParameter(name='save_period', value=1)
184 250
185 251 if save==1:
186 252 opObj10 = procUnitConfObjC.addOperation(name='HDFWriter')
187 253 opObj10.addParameter(name='path',value=path_save)
188 254 #opObj10.addParameter(name='mode',value=0)
189 255 opObj10.addParameter(name='blocksPerFile',value='360',format='int')
190 256 #opObj10.addParameter(name='metadataList',value='utctimeInit,heightList,nIncohInt,nCohInt,nProfiles,channelList',format='list')#profileIndex
191 257 opObj10.addParameter(name='metadataList',value='utctimeInit,heightList,nIncohInt,nCohInt,nProfiles,channelList',format='list')#profileIndex
192 258 opObj10.addParameter(name='dataList',value='data_pow,data_dop,utctime',format='list')#,format='list'
193 259
194 260 if integration==1:
195 261 V=10
196 262 blocksPerfile=360
197 263 print("* Velocidad del Pedestal:",V)
198 264 tmp_blocksPerfile = 100
199 265 f_a_p= int(tmp_blocksPerfile/V)
200 266
201 267 opObj11 = procUnitConfObjC.addOperation(name='PedestalInformation')
202 268 opObj11.addParameter(name='path_ped', value=path_ped)
203 269 #opObj11.addParameter(name='path_adq', value=path_adq)
204 270 opObj11.addParameter(name='t_Interval_p', value='0.01', format='float')
205 271 opObj11.addParameter(name='blocksPerfile', value=blocksPerfile, format='int')
206 272 opObj11.addParameter(name='n_Muestras_p', value='100', format='float')
207 273 opObj11.addParameter(name='f_a_p', value=f_a_p, format='int')
208 274 opObj11.addParameter(name='online', value='0', format='int')
209 275
210 276 opObj11 = procUnitConfObjC.addOperation(name='Block360')
211 277 opObj11.addParameter(name='n', value='10', format='int')
212 278 opObj11.addParameter(name='mode', value=mode, format='int')
213 279
214 280 # este bloque funciona bien con divisores de 360 no olvidar 0 10 20 30 40 60 90 120 180
215 281 opObj11= procUnitConfObjC.addOperation(name='WeatherPlot',optype='other')
216 282 controllerObj.start()
Show file before | Show file after | Hide comments
@@ -1,217 +1,269
1 1 # Ing. AVP
2 2 # 06/10/2021
3 3 # ARCHIVO DE LECTURA
4 4 import os, sys
5 5 import datetime
6 6 import time
7 7 from schainpy.controller import Project
8 8 #### NOTA###########################################
9 9 # INPUT :
10 10 # VELOCIDAD PARAMETRO : V = 2Β°/seg
11 11 # MODO PULSE PAIR O MOMENTOS: 0 : Pulse Pair ,1 : Momentos
12 12 ######################################################
13 13 ##### PROCESAMIENTO ##################################
14 14 ##### OJO TENER EN CUENTA EL n= para el Pulse Pair ##
15 15 ##### O EL n= nFFTPoints ###
16 16 ######################################################
17 17 ######## BUSCAMOS EL numero de IPP equivalente 1Β°#####
18 18 ######## Sea V la velocidad del Pedestal en Β°/seg#####
19 19 ######## 1Β° sera Recorrido en un tiempo de 1/V ######
20 20 ######## IPP del Radar 400 useg --> 60 Km ############
21 21 ######## n = 1/(V(Β°/seg)*IPP(Km)) , NUMERO DE IPP ##
22 22 ######## n = 1/(V*IPP) #############################
23 23 ######## VELOCIDAD DEL PEDESTAL ######################
24 24 print("SETUP- RADAR METEOROLOGICO")
25 V = 6
25 26 V = 10
26 mode = 0
27 #path = '/DATA_RM/23/6v'
28 #path = '/DATA_RM/TEST_INTEGRACION_2M'
29 path = '/DATA_RM/WR_20_OCT'
27 mode = 1
28 #--------------------------PATH -----------------------------
29 # path = '/DATA_RM/23/6v'
30 # path = '/DATA_RM/TEST_INTEGRACION_2M'
31 # path = '/DATA_RM/TEST_19OCTUBRE/10MHZ'
32 # path = '/DATA_RM/WR_20_OCT'
33 #### path_ped='/DATA_RM/TEST_PEDESTAL/P20211012-082745'
34 ####path_ped='/DATA_RM/TEST_PEDESTAL/P20211019-192244'
35 path ="/DATA_RM/10"
36 # path = '/DATA_RM/WR_POT_09_1'
37 #path ="/DATA_RM/11"
38 #path ="/DATA_RM/11"
39 #-------------------------PATH-PLOTEO------------------------------------
40 #figpath_pp = "/home/soporte/Pictures/TEST_PP"
41 #figpath_spec = "/home/soporte/Pictures/TEST_MOM"
42 figpath_spec = "/home/soporte/Pictures/ppi"
43 figpath_ppi = "/home/soporte/Pictures/ppi_SPEC_10DIC"
44 figpath_ppi_pp = "/home/soporte/Pictures/ppi_2_10"
45 figpath_pp = "/home/soporte/Pictures/TEST_POT"
46 figpath_pp = "/home/soporte/Pictures/TEST_POT2"
30 47
31 48 #path_ped='/DATA_RM/TEST_PEDESTAL/P20211012-082745'
32 path_ped='/DATA_RM/TEST_PEDESTAL/P20211020-131248'
49 #path_ped='/DATA_RM/TEST_PEDESTAL/P20211020-131248'
50 #path_ped='/DATA_RM/TEST_PEDESTAL/P20211110-171003'
51 #path_ped='/DATA_RM/TEST_PEDESTAL/P20211111-173856'
52 path_ped = "/DATA_RM/TEST_PEDESTAL/P20211110-171003"
33 53
34 figpath_pp = "/home/soporte/Pictures/TEST_PP"
54
55 figpath_pp = "/home/soporte/Pictures/25TEST_PP"
35 56 figpath_mom = "/home/soporte/Pictures/TEST_MOM"
57 #--------------------------OPCIONES-----------------------------------
36 58 plot = 0
37 59 integration = 1
38 save = 0
60 save = 1
39 61 if save == 1:
40 62 if mode==0:
41 63 path_save = '/DATA_RM/TEST_HDF5_PP_23/6v'
42 64 path_save = '/DATA_RM/TEST_HDF5_PP'
43 65 path_save = '/DATA_RM/TEST_HDF5_PP_100'
66 path_save = '/DATA_RM/TEST_EMPTHY'
44 67 else:
45 68 path_save = '/DATA_RM/TEST_HDF5_SPEC_23_V2/6v'
69 path_save = '/DATA_RM/TEST_EMPTHY_SPEC'
70 path_save = '/DATA_RM/LAST_TEST_16_VACIO3'
71 path_save = '/DATA_RM/LAST_TEST_30_360'
46 72
47 73 print("* PATH data ADQ :", path)
48 74 print("* Velocidad Pedestal :",V,"Β°/seg")
49 75 ############################ NRO Perfiles PROCESAMIENTO ###################
50 76 V=V
51 77 IPP=400*1e-6
52 78 n= int(1/(V*IPP))
53 79 print("* n - NRO Perfiles Proc:", n )
54 80 ################################## MODE ###################################
55 81 print("* Modo de Operacion :",mode)
56 82 if mode ==0:
57 83 print("* Met. Seleccionado : Pulse Pair")
58 84 else:
59 85 print("* Met. Momentos : Momentos")
60 86
61 87 ################################## MODE ###################################
62 88 print("* Grabado de datos :",save)
63 89 if save ==1:
64 90 if mode==0:
65 91 ope= "Pulse Pair"
66 92 else:
67 93 ope= "Momentos"
68 94 print("* Path-Save Data -", ope , path_save)
69 95
70 96 print("* Integracion de datos :",integration)
71 97
72 time.sleep(5)
98 time.sleep(3)
73 99 #remotefolder = "/home/wmaster/graficos"
74 100 #######################################################################
75 101 ################# RANGO DE PLOTEO######################################
76 102 dBmin = '1'
77 103 dBmax = '85'
78 xmin = '15'
79 xmax = '15.25'
104 xmin = '17'
105 xmax = '17.25'
80 106 ymin = '0'
81 107 ymax = '600'
82 108 #######################################################################
83 109 ########################FECHA##########################################
84 110 str = datetime.date.today()
85 111 today = str.strftime("%Y/%m/%d")
86 112 str2 = str - datetime.timedelta(days=1)
87 113 yesterday = str2.strftime("%Y/%m/%d")
88 114 #######################################################################
89 115 ########################SIGNAL CHAIN ##################################
90 116 #######################################################################
91 117 desc = "USRP_test"
92 118 filename = "USRP_processing.xml"
93 119 controllerObj = Project()
94 120 controllerObj.setup(id = '191', name='Test_USRP', description=desc)
95 121 #######################################################################
96 122 ######################## UNIDAD DE LECTURA#############################
97 123 #######################################################################
98 124 readUnitConfObj = controllerObj.addReadUnit(datatype='DigitalRFReader',
99 125 path=path,
100 startDate="2021/01/01",#today,
126 startDate="2021/11/10",#today,
101 127 endDate="2021/12/30",#today,
102 startTime='00:00:00',
128 startTime='17:10:25',
103 129 endTime='23:59:59',
104 130 delay=0,
105 131 #set=0,
106 132 online=0,
107 133 walk=1,
108 134 ippKm = 60)
109 135
110 136 opObj11 = readUnitConfObj.addOperation(name='printInfo')
111 137
112 procUnitConfObjA = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
138 procUnitConfObjA = controllerObj.addProcUnit(datatype='VoltageProc',inputId=readUnitConfObj.getId())
139
140
141 #opObj11 = procUnitConfObjA.addOperation(name='setH0')
142 #opObj11.addParameter(name='h0', value='-2.8', format='float')
143
144 opObj11 = procUnitConfObjA.addOperation(name='selectHeights')
145 opObj11.addParameter(name='minIndex', value='1', format='int')
146 # opObj11.addParameter(name='maxIndex', value='10000', format='int')
147 opObj11.addParameter(name='maxIndex', value='400', format='int')
113 148
114 149 if mode ==0:
115 150 ####################### METODO PULSE PAIR ######################################################################
116 151 opObj11 = procUnitConfObjA.addOperation(name='PulsePair', optype='other')
117 152 opObj11.addParameter(name='n', value=int(n), format='int')#10 VOY A USAR 250 DADO QUE LA VELOCIDAD ES 10 GRADOS
118 153 #opObj11.addParameter(name='removeDC', value=1, format='int')
119 154 ####################### METODO Parametros ######################################################################
120 155 procUnitConfObjB= controllerObj.addProcUnit(datatype='ParametersProc',inputId=procUnitConfObjA.getId())
121 156 if plot==1:
122 157 opObj11 = procUnitConfObjB.addOperation(name='GenericRTIPlot',optype='external')
123 158 opObj11.addParameter(name='attr_data', value='dataPP_POW')
124 159 opObj11.addParameter(name='colormap', value='jet')
125 160 opObj11.addParameter(name='xmin', value=xmin)
126 161 opObj11.addParameter(name='xmax', value=xmax)
127 162 opObj11.addParameter(name='zmin', value=dBmin)
128 163 opObj11.addParameter(name='zmax', value=dBmax)
129 164 opObj11.addParameter(name='save', value=figpath_pp)
130 165 opObj11.addParameter(name='showprofile', value=0)
131 166 opObj11.addParameter(name='save_period', value=50)
132 167
133 168 ####################### METODO ESCRITURA #######################################################################
134 if save==1:
135 opObj10 = procUnitConfObjB.addOperation(name='HDFWriter')
136 opObj10.addParameter(name='path',value=path_save)
137 #opObj10.addParameter(name='mode',value=0)
138 opObj10.addParameter(name='blocksPerFile',value='100',format='int')
139 opObj10.addParameter(name='metadataList',value='utctimeInit,timeZone,paramInterval,profileIndex,channelList,heightList,flagDataAsBlock',format='list')
140 opObj10.addParameter(name='dataList',value='dataPP_POW,dataPP_DOP,utctime',format='list')#,format='list'
169
141 170 if integration==1:
142 V=10
143 blocksPerfile=360
171 V=V
172 blocksPerfile=100
144 173 print("* Velocidad del Pedestal:",V)
145 174 tmp_blocksPerfile = 100
146 175 f_a_p= int(tmp_blocksPerfile/V)
147 176
148 177 opObj11 = procUnitConfObjB.addOperation(name='PedestalInformation')
149 178 opObj11.addParameter(name='path_ped', value=path_ped)
150 179 #opObj11.addParameter(name='path_adq', value=path_adq)
151 180 opObj11.addParameter(name='t_Interval_p', value='0.01', format='float')
152 181 opObj11.addParameter(name='blocksPerfile', value=blocksPerfile, format='int')
153 182 opObj11.addParameter(name='n_Muestras_p', value='100', format='float')
154 183 opObj11.addParameter(name='f_a_p', value=f_a_p, format='int')
155 184 opObj11.addParameter(name='online', value='0', format='int')
156
185 '''
157 186 opObj11 = procUnitConfObjB.addOperation(name='Block360')
158 187 opObj11.addParameter(name='n', value='10', format='int')
159 188 opObj11.addParameter(name='mode', value=mode, format='int')
160
189 '''
161 190 # este bloque funciona bien con divisores de 360 no olvidar 0 10 20 30 40 60 90 120 180
162
191 '''
163 192 opObj11= procUnitConfObjB.addOperation(name='WeatherPlot',optype='other')
164
193 opObj11.addParameter(name='save', value=figpath_ppi_pp)
194 opObj11.addParameter(name='save_period', value=1)
195 '''
196 if save==1:
197 opObj10 = procUnitConfObjB.addOperation(name='HDFWriter')
198 opObj10.addParameter(name='path',value=path_save)
199 #opObj10.addParameter(name='mode',value=0)
200 opObj10.addParameter(name='blocksPerFile',value='100',format='int')
201 opObj10.addParameter(name='metadataList',value='utctimeInit,timeZone,paramInterval,profileIndex,channelList,heightList,flagDataAsBlock',format='list')
202 opObj10.addParameter(name='dataList',value='dataPP_POW,dataPP_DOP,azimuth,utctime',format='list')#,format='list'
165 203
166 204 else:
167 205 ####################### METODO SPECTROS ######################################################################
168 206 procUnitConfObjB = controllerObj.addProcUnit(datatype='SpectraProc', inputId=procUnitConfObjA.getId())
169 207 procUnitConfObjB.addParameter(name='nFFTPoints', value=n, format='int')
170 208 procUnitConfObjB.addParameter(name='nProfiles' , value=n, format='int')
171 209
172 210 procUnitConfObjC = controllerObj.addProcUnit(datatype='ParametersProc',inputId=procUnitConfObjB.getId())
173 211 procUnitConfObjC.addOperation(name='SpectralMoments')
174 212 if plot==1:
175 213 dBmin = '1'
176 214 dBmax = '65'
177 215 opObj11 = procUnitConfObjC.addOperation(name='PowerPlot',optype='external')
178 216 opObj11.addParameter(name='xmin', value=xmin)
179 217 opObj11.addParameter(name='xmax', value=xmax)
180 218 opObj11.addParameter(name='zmin', value=dBmin)
181 219 opObj11.addParameter(name='zmax', value=dBmax)
182 220 opObj11.addParameter(name='save', value=figpath_mom)
183 221 opObj11.addParameter(name='showprofile', value=0)
184 222 opObj11.addParameter(name='save_period', value=100)
185 223
186 if save==1:
187 opObj10 = procUnitConfObjC.addOperation(name='HDFWriter')
188 opObj10.addParameter(name='path',value=path_save)
189 #opObj10.addParameter(name='mode',value=0)
190 opObj10.addParameter(name='blocksPerFile',value='360',format='int')
191 #opObj10.addParameter(name='metadataList',value='utctimeInit,heightList,nIncohInt,nCohInt,nProfiles,channelList',format='list')#profileIndex
192 opObj10.addParameter(name='metadataList',value='utctimeInit,heightList,nIncohInt,nCohInt,nProfiles,channelList',format='list')#profileIndex
193 opObj10.addParameter(name='dataList',value='data_pow,data_dop,utctime',format='list')#,format='list'
194 224
195 225 if integration==1:
196 V=10
197 blocksPerfile=360
226 V=V
227 blocksPerfile=100
198 228 print("* Velocidad del Pedestal:",V)
199 229 tmp_blocksPerfile = 100
200 230 f_a_p= int(tmp_blocksPerfile/V)
201 231
202 232 opObj11 = procUnitConfObjC.addOperation(name='PedestalInformation')
203 233 opObj11.addParameter(name='path_ped', value=path_ped)
204 234 #opObj11.addParameter(name='path_adq', value=path_adq)
205 235 opObj11.addParameter(name='t_Interval_p', value='0.01', format='float')
206 236 opObj11.addParameter(name='blocksPerfile', value=blocksPerfile, format='int')
207 237 opObj11.addParameter(name='n_Muestras_p', value='100', format='float')
208 238 opObj11.addParameter(name='f_a_p', value=f_a_p, format='int')
209 239 opObj11.addParameter(name='online', value='0', format='int')
210 240
211 241 opObj11 = procUnitConfObjC.addOperation(name='Block360')
212 opObj11.addParameter(name='n', value='10', format='int')
242 opObj11.addParameter(name='n', value='1', format='int')
213 243 opObj11.addParameter(name='mode', value=mode, format='int')
244 '''
245 # este bloque funciona bien con divisores de 360 no olvidar 0 10 20 30 40 60 90 120 180
246 opObj11= procUnitConfObjC.addOperation(name='WeatherPlot',optype='other')
247 opObj11.addParameter(name='save', value=figpath_ppi)
248 opObj11.addParameter(name='save_period', value=1)
249 '''
250
251 if save==1:
252 opObj10 = procUnitConfObjC.addOperation(name='HDFWriter')
253 opObj10.addParameter(name='path',value=path_save)
254 opObj10.addParameter(name='mode',value='weather')
255 opObj10.addParameter(name='blocksPerFile',value='360',format='int')
256 #opObj10.addParameter(name='metadataList',value='utctimeInit,heightList,nIncohInt,nCohInt,nProfiles,channelList',format='list')#profileIndex
257 opObj10.addParameter(name='metadataList',value='utctimeInit,heightList,nIncohInt,nCohInt,nProfiles,channelList',format='list')#profileIndex
258 #opObj10.addParameter(name='dataList',value='data_pow,data_dop,azimuth,utctime',format='list')#,format='list'
259 opObj10.addParameter(name='dataList',value='data_360,data_azi,utctime',format='list')#,format='list'
260
261
214 262
263 '''
215 264 # este bloque funciona bien con divisores de 360 no olvidar 0 10 20 30 40 60 90 120 180
216 265 opObj11= procUnitConfObjC.addOperation(name='WeatherPlot',optype='other')
266 opObj11.addParameter(name='save', value=figpath_ppi)
267 opObj11.addParameter(name='save_period', value=1)
268 '''
217 269 controllerObj.start()
Show file before | Show file after | Hide comments
@@ -1,119 +1,122
1 1 # Ing-AlexanderValdez
2 2 # Monitoreo de Pedestal
3 3
4 4 ############## IMPORTA LIBRERIAS ###################
5 5 import os,numpy,h5py
6 6 import sys,time
7 7 import matplotlib.pyplot as plt
8 8 ####################################################
9 9 #################################################################
10 10 # LA FECHA 21-10-20 CORRESPONDE A LAS PRUEBAS DEL DIA MIERCOLES
11 11 # 1:15:51 pm hasta 3:49:32 pm
12 12 #################################################################
13 13
14 14 #path_ped = '/DATA_RM/TEST_PEDESTAL/P20211012-082745'
15 15 path_ped = '/DATA_RM/TEST_PEDESTAL/P20211020-131248'
16 path_ped = '/DATA_RM/TEST_PEDESTAL/P20211110-171003'
17 path_ped = '/DATA_RM/TEST_PEDESTAL/P20211111-173856'
18 #path_ped = '/DATA_RM/TEST_PEDESTAL/P20211111-173409'
16 19 # Metodo para verificar numero
17 20 def isNumber(str):
18 21 try:
19 22 float(str)
20 23 return True
21 24 except:
22 25 return False
23 26 # Metodo para extraer el arreglo
24 27 def getDatavaluefromDirFilename(path,file,value):
25 28 dir_file= path+"/"+file
26 29 fp = h5py.File(dir_file,'r')
27 30 array = fp['Data'].get(value)[()]
28 31 fp.close()
29 32 return array
30 33
31 34 # LISTA COMPLETA DE ARCHIVOS HDF5 Pedestal
32 35 LIST= sorted(os.listdir(path_ped))
33 36 m=len(LIST)
34 37 print("TOTAL DE ARCHIVOS DE PEDESTAL:",m)
35 38 # Contadores temporales
36 39 k= 0
37 40 l= 0
38 41 t= 0
39 42 # Marca de tiempo temporal
40 43 time_ = numpy.zeros([m])
41 44 # creacion de
42 45 for i in range(m):
43 46 print("order:",i)
44 47 tmp_azi_pos = getDatavaluefromDirFilename(path=path_ped,file=LIST[i],value="azi_pos")
45 48 tmp_ele_pos = getDatavaluefromDirFilename(path=path_ped,file=LIST[i],value="ele_pos")
46 49 tmp_azi_vel = getDatavaluefromDirFilename(path=path_ped,file=LIST[i],value="azi_vel")
47 50 tmp_ele_vel = getDatavaluefromDirFilename(path=path_ped,file=LIST[i],value="azi_vel")# nuevo :D
48 51
49 52 time_[i] = getDatavaluefromDirFilename(path=path_ped,file=LIST[i],value="utc")
50 53
51 54 k=k +tmp_azi_pos.shape[0]
52 55 l=l +tmp_ele_pos.shape[0]
53 56 t=t +tmp_azi_vel.shape[0]
54 57
55 58 print("TOTAL DE MUESTRAS, ARCHIVOS X100:",k)
56 59 time.sleep(5)
57 60 ######CREACION DE ARREGLOS CANTIDAD DE VALORES POR MUESTRA#################
58 61 azi_pos = numpy.zeros([k])
59 62 ele_pos = numpy.zeros([l])
60 63 time_azi_pos= numpy.zeros([k])
61 64 # Contadores temporales
62 65 p=0
63 66 r=0
64 67 z=0
65 68 # VARIABLES TMP para almacenar azimuth, elevacion y tiempo
66 69
67 70 #for filename in sorted(os.listdir(path_ped)):
68 71 # CONDICION POR LEER EN TIEMPO REAL NO OFFLINE
69 72
70 73 for filename in LIST:
71 74 tmp_azi_pos = getDatavaluefromDirFilename(path=path_ped,file=filename,value="azi_pos")
72 75 tmp_ele_pos = getDatavaluefromDirFilename(path=path_ped,file=filename,value="ele_pos")
73 76 # CONDICION POR LEER EN TIEMPO REAL NO OFFLINE
74 77
75 78 if z==(m-1):
76 79 tmp_azi_time=numpy.arange(time_[z],time_[z]+1,1/(tmp_azi_pos.shape[0]))
77 80 else:
78 81 tmp_azi_time=numpy.arange(time_[z],time_[z+1],(time_[z+1]-time_[z])/(tmp_azi_pos.shape[0]))
79 82
80 83 print(filename,time_[z])
81 84 print(z,tmp_azi_pos.shape[0])
82 85
83 86 i=0
84 87 for i in range(tmp_azi_pos.shape[0]):
85 88 index=p+i
86 89 azi_pos[index]=tmp_azi_pos[i]
87 90 time_azi_pos[index]=tmp_azi_time[i]
88 91 p=p+tmp_azi_pos.shape[0]
89 92 i=0
90 93 for i in range(tmp_ele_pos.shape[0]):
91 94 index=r+i
92 95 ele_pos[index]=tmp_ele_pos[i]
93 96 r=r+tmp_ele_pos.shape[0]
94 97
95 98
96 99 z+=1
97 100
98 101
99 102 ######## GRAFIQUEMOS Y VEAMOS LOS DATOS DEL Pedestal
100 103 fig, ax = plt.subplots(figsize=(16,8))
101 104 print(time_azi_pos.shape)
102 105 print(azi_pos.shape)
103 106 t=numpy.arange(time_azi_pos.shape[0])*0.01/(60.0)
104 107 plt.plot(t,azi_pos,label='AZIMUTH_POS',color='blue')
105 108
106 109 # AQUI ESTOY ADICIONANDO LA POSICION EN elevaciont=numpy.arange(len(ele_pos))*0.01/60.0
107 110 t=numpy.arange(len(ele_pos))*0.01/60.0
108 111 plt.plot(t,ele_pos,label='ELEVATION_POS',color='red')#*10
109 112
110 113 #ax.set_xlim(0, 9)
111 ax.set_ylim(-5, 400)
114 ax.set_ylim(-5, 20)
112 115 plt.ylabel("Azimuth Position")
113 116 plt.xlabel("Muestra")
114 117 plt.title('Azimuth Position vs Muestra ', fontsize=20)
115 118 axes = plt.gca()
116 119 axes.yaxis.grid()
117 120 plt.xticks(fontsize=16)
118 121 plt.yticks(fontsize=16)
119 122 plt.show()
General Comments 0
You need to be logged in to leave comments. Login now