##// END OF EJS Templates
schain
RSS
cambios para amisr ISR
joabAM -
r1465:65c0d2b45bc1
parent child
Show More
Side by Side Unified
Context file:
r1465:65c0d2b45bc1 Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -0,0 +1,132
1 import numpy
2
3 def setConstants(dataOut):
4 dictionary = {}
5 dictionary["M"] = dataOut.normFactor
6 dictionary["N"] = dataOut.nFFTPoints
7 dictionary["ippSeconds"] = dataOut.ippSeconds
8 dictionary["K"] = dataOut.nIncohInt
9
10 return dictionary
11
12 def initialValuesFunction(data_spc, constants):
13 #Constants
14 M = constants["M"]
15 N = constants["N"]
16 ippSeconds = constants["ippSeconds"]
17
18 S1 = data_spc[0,:]/(N*M)
19 S2 = data_spc[1,:]/(N*M)
20
21 Nl=min(S1)
22 A=sum(S1-Nl)/N
23 #x = dataOut.getVelRange() #below matches Madrigal data better
24 x=numpy.linspace(-(N/2)/(N*ippSeconds),(N/2-1)/(N*ippSeconds),N)*-(6.0/2)
25 v=sum(x*(S1-Nl))/sum(S1-Nl)
26 al1=numpy.sqrt(sum(x**2*(S1-Nl))/sum(S2-Nl)-v**2)
27 p0=[al1,A,A,v,min(S1),min(S2)]#first guess(width,amplitude,velocity,noise)
28 return p0
29
30 def modelFunction(p, constants):
31 ippSeconds = constants["ippSeconds"]
32 N = constants["N"]
33
34 fm_c = ACFtoSPC(p, constants)
35 fm = numpy.hstack((fm_c[0],fm_c[1]))
36 return fm
37
38 def errorFunction(p, constants, LT):
39
40 J=makeJacobian(p, constants)
41 J =numpy.dot(LT,J)
42 covm =numpy.linalg.inv(numpy.dot(J.T ,J))
43 #calculate error as the square root of the covariance matrix diagonal
44 #multiplying by 1.96 would give 95% confidence interval
45 err =numpy.sqrt(numpy.diag(covm))
46 return err
47
48 #-----------------------------------------------------------------------------------
49
50 def ACFw(alpha,A1,A2,vd,x,N,ippSeconds):
51 #creates weighted autocorrelation function based on the operational model
52 #x is n or N-n
53 k=2*numpy.pi/3.0
54 pdt=x*ippSeconds
55 #both correlated channels ACFs are created at the sametime
56 R1=A1*numpy.exp(-1j*k*vd*pdt)/numpy.sqrt(1+(alpha*k*pdt)**2)
57 R2=A2*numpy.exp(-1j*k*vd*pdt)/numpy.sqrt(1+(alpha*k*pdt)**2)
58 # T is the triangle weigthing function
59 T=1-abs(x)/N
60 Rp1=T*R1
61 Rp2=T*R2
62 return [Rp1,Rp2]
63
64 def ACFtoSPC(p, constants):
65 #calls the create ACF function and transforms the ACF to spectra
66 N = constants["N"]
67 ippSeconds = constants["ippSeconds"]
68
69 n=numpy.linspace(0,(N-1),N)
70 Nn=N-n
71 R = ACFw(p[0],p[1],p[2],p[3],n,N,ippSeconds)
72 RN = ACFw(p[0],p[1],p[2],p[3],Nn,N,ippSeconds)
73 Rf1=R[0]+numpy.conjugate(RN[0])
74 Rf2=R[1]+numpy.conjugate(RN[1])
75 sw1=numpy.fft.fft(Rf1,n=N)
76 sw2=numpy.fft.fft(Rf2,n=N)
77 #the fft needs to be shifted, noise added, and takes only the real part
78 sw0=numpy.real(numpy.fft.fftshift(sw1))+abs(p[4])
79 sw1=numpy.real(numpy.fft.fftshift(sw2))+abs(p[5])
80 return [sw0,sw1]
81
82 def makeJacobian(p, constants):
83 #create Jacobian matrix
84 N = constants["N"]
85 IPPt = constants["ippSeconds"]
86
87 n=numpy.linspace(0,(N-1),N)
88 Nn=N-n
89 k=2*numpy.pi/3.0
90 #created weighted ACF
91 R=ACFw(p[0],p[1],p[2],p[3],n,N,IPPt)
92 RN=ACFw(p[0],p[1],p[2],p[3],Nn,N,IPPt)
93 #take derivatives with respect to the fit parameters
94 Jalpha1=R[0]*-1*(k*n*IPPt)**2*p[0]/(1+(p[0]*k*n*IPPt)**2)+numpy.conjugate(RN[0]*-1*(k*Nn*IPPt)**2*p[0]/(1+(p[0]*k*Nn*IPPt)**2))
95 Jalpha2=R[1]*-1*(k*n*IPPt)**2*p[0]/(1+(p[0]*k*n*IPPt)**2)+numpy.conjugate(RN[1]*-1*(k*Nn*IPPt)**2*p[0]/(1+(p[0]*k*Nn*IPPt)**2))
96 JA1=R[0]/p[1]+numpy.conjugate(RN[0]/p[1])
97 JA2=R[1]/p[2]+numpy.conjugate(RN[1]/p[2])
98 Jvd1=R[0]*-1j*k*n*IPPt+numpy.conjugate(RN[0]*-1j*k*Nn*IPPt)
99 Jvd2=R[1]*-1j*k*n*IPPt+numpy.conjugate(RN[1]*-1j*k*Nn*IPPt)
100 #fft
101 sJalp1=numpy.fft.fft(Jalpha1,n=N)
102 sJalp2=numpy.fft.fft(Jalpha2,n=N)
103 sJA1=numpy.fft.fft(JA1,n=N)
104 sJA2=numpy.fft.fft(JA2,n=N)
105 sJvd1=numpy.fft.fft(Jvd1,n=N)
106 sJvd2=numpy.fft.fft(Jvd2,n=N)
107 sJalp1=numpy.real(numpy.fft.fftshift(sJalp1))
108 sJalp2=numpy.real(numpy.fft.fftshift(sJalp2))
109 sJA1=numpy.real(numpy.fft.fftshift(sJA1))
110 sJA2=numpy.real(numpy.fft.fftshift(sJA2))
111 sJvd1=numpy.real(numpy.fft.fftshift(sJvd1))
112 sJvd2=numpy.real(numpy.fft.fftshift(sJvd2))
113 sJnoise=numpy.ones(numpy.shape(sJvd1))
114 #combine arrays
115 za=numpy.zeros([N])
116 sJalp=zip(sJalp1,sJalp2)
117 sJA1=zip(sJA1,za)
118 sJA2=zip(za,sJA2)
119 sJvd=zip(sJvd1,sJvd2)
120 sJn1=zip(sJnoise, za)
121 sJn2=zip(za, sJnoise)
122 #reshape from 2D to 1D
123 sJalp=numpy.reshape(list(sJalp), [2*N])
124 sJA1=numpy.reshape(list(sJA1), [2*N])
125 sJA2=numpy.reshape(list(sJA2), [2*N])
126 sJvd=numpy.reshape(list(sJvd), [2*N])
127 sJn1=numpy.reshape(list(sJn1), [2*N])
128 sJn2=numpy.reshape(list(sJn2), [2*N])
129 #combine into matrix and transpose
130 J=numpy.array([sJalp,sJA1,sJA2,sJvd,sJn1,sJn2])
131 J=J.T
132 return J
Show file before | Show file after | Hide comments
@@ -1,699 +1,701
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """Base class to create plot operations
5 """Base class to create plot operations
6
6
7 """
7 """
8
8
9 import os
9 import os
10 import sys
10 import sys
11 import zmq
11 import zmq
12 import time
12 import time
13 import numpy
13 import numpy
14 import datetime
14 import datetime
15 from collections import deque
15 from collections import deque
16 from functools import wraps
16 from functools import wraps
17 from threading import Thread
17 from threading import Thread
18 import matplotlib
18 import matplotlib
19
19
20 if 'BACKEND' in os.environ:
20 if 'BACKEND' in os.environ:
21 matplotlib.use(os.environ['BACKEND'])
21 matplotlib.use(os.environ['BACKEND'])
22 elif 'linux' in sys.platform:
22 elif 'linux' in sys.platform:
23 matplotlib.use("TkAgg")
23 matplotlib.use("TkAgg")
24 elif 'darwin' in sys.platform:
24 elif 'darwin' in sys.platform:
25 matplotlib.use('MacOSX')
25 matplotlib.use('MacOSX')
26 else:
26 else:
27 from schainpy.utils import log
27 from schainpy.utils import log
28 log.warning('Using default Backend="Agg"', 'INFO')
28 log.warning('Using default Backend="Agg"', 'INFO')
29 matplotlib.use('Agg')
29 matplotlib.use('Agg')
30
30
31 import matplotlib.pyplot as plt
31 import matplotlib.pyplot as plt
32 from matplotlib.patches import Polygon
32 from matplotlib.patches import Polygon
33 from mpl_toolkits.axes_grid1 import make_axes_locatable
33 from mpl_toolkits.axes_grid1 import make_axes_locatable
34 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
34 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
35
35
36 from schainpy.model.data.jrodata import PlotterData
36 from schainpy.model.data.jrodata import PlotterData
37 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
37 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
38 from schainpy.utils import log
38 from schainpy.utils import log
39
39
40 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
40 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
41 blu_values = matplotlib.pyplot.get_cmap(
41 blu_values = matplotlib.pyplot.get_cmap(
42 'seismic_r', 20)(numpy.arange(20))[10:15]
42 'seismic_r', 20)(numpy.arange(20))[10:15]
43 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
43 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
44 'jro', numpy.vstack((blu_values, jet_values)))
44 'jro', numpy.vstack((blu_values, jet_values)))
45 matplotlib.pyplot.register_cmap(cmap=ncmap)
45 matplotlib.pyplot.register_cmap(cmap=ncmap)
46
46
47 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis',
47 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis',
48 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
48 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
49
49
50 EARTH_RADIUS = 6.3710e3
50 EARTH_RADIUS = 6.3710e3
51
51
52 def ll2xy(lat1, lon1, lat2, lon2):
52 def ll2xy(lat1, lon1, lat2, lon2):
53
53
54 p = 0.017453292519943295
54 p = 0.017453292519943295
55 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
55 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
56 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
56 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
57 r = 12742 * numpy.arcsin(numpy.sqrt(a))
57 r = 12742 * numpy.arcsin(numpy.sqrt(a))
58 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
58 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
59 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
59 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
60 theta = -theta + numpy.pi/2
60 theta = -theta + numpy.pi/2
61 return r*numpy.cos(theta), r*numpy.sin(theta)
61 return r*numpy.cos(theta), r*numpy.sin(theta)
62
62
63
63
64 def km2deg(km):
64 def km2deg(km):
65 '''
65 '''
66 Convert distance in km to degrees
66 Convert distance in km to degrees
67 '''
67 '''
68
68
69 return numpy.rad2deg(km/EARTH_RADIUS)
69 return numpy.rad2deg(km/EARTH_RADIUS)
70
70
71
71
72 def figpause(interval):
72 def figpause(interval):
73 backend = plt.rcParams['backend']
73 backend = plt.rcParams['backend']
74 if backend in matplotlib.rcsetup.interactive_bk:
74 if backend in matplotlib.rcsetup.interactive_bk:
75 figManager = matplotlib._pylab_helpers.Gcf.get_active()
75 figManager = matplotlib._pylab_helpers.Gcf.get_active()
76 if figManager is not None:
76 if figManager is not None:
77 canvas = figManager.canvas
77 canvas = figManager.canvas
78 if canvas.figure.stale:
78 if canvas.figure.stale:
79 canvas.draw()
79 canvas.draw()
80 try:
80 try:
81 canvas.start_event_loop(interval)
81 canvas.start_event_loop(interval)
82 except:
82 except:
83 pass
83 pass
84 return
84 return
85
85
86 def popup(message):
86 def popup(message):
87 '''
87 '''
88 '''
88 '''
89
89
90 fig = plt.figure(figsize=(12, 8), facecolor='r')
90 fig = plt.figure(figsize=(12, 8), facecolor='r')
91 text = '\n'.join([s.strip() for s in message.split(':')])
91 text = '\n'.join([s.strip() for s in message.split(':')])
92 fig.text(0.01, 0.5, text, ha='left', va='center',
92 fig.text(0.01, 0.5, text, ha='left', va='center',
93 size='20', weight='heavy', color='w')
93 size='20', weight='heavy', color='w')
94 fig.show()
94 fig.show()
95 figpause(1000)
95 figpause(1000)
96
96
97
97
98 class Throttle(object):
98 class Throttle(object):
99 '''
99 '''
100 Decorator that prevents a function from being called more than once every
100 Decorator that prevents a function from being called more than once every
101 time period.
101 time period.
102 To create a function that cannot be called more than once a minute, but
102 To create a function that cannot be called more than once a minute, but
103 will sleep until it can be called:
103 will sleep until it can be called:
104 @Throttle(minutes=1)
104 @Throttle(minutes=1)
105 def foo():
105 def foo():
106 pass
106 pass
107
107
108 for i in range(10):
108 for i in range(10):
109 foo()
109 foo()
110 print "This function has run %s times." % i
110 print "This function has run %s times." % i
111 '''
111 '''
112
112
113 def __init__(self, seconds=0, minutes=0, hours=0):
113 def __init__(self, seconds=0, minutes=0, hours=0):
114 self.throttle_period = datetime.timedelta(
114 self.throttle_period = datetime.timedelta(
115 seconds=seconds, minutes=minutes, hours=hours
115 seconds=seconds, minutes=minutes, hours=hours
116 )
116 )
117
117
118 self.time_of_last_call = datetime.datetime.min
118 self.time_of_last_call = datetime.datetime.min
119
119
120 def __call__(self, fn):
120 def __call__(self, fn):
121 @wraps(fn)
121 @wraps(fn)
122 def wrapper(*args, **kwargs):
122 def wrapper(*args, **kwargs):
123 coerce = kwargs.pop('coerce', None)
123 coerce = kwargs.pop('coerce', None)
124 if coerce:
124 if coerce:
125 self.time_of_last_call = datetime.datetime.now()
125 self.time_of_last_call = datetime.datetime.now()
126 return fn(*args, **kwargs)
126 return fn(*args, **kwargs)
127 else:
127 else:
128 now = datetime.datetime.now()
128 now = datetime.datetime.now()
129 time_since_last_call = now - self.time_of_last_call
129 time_since_last_call = now - self.time_of_last_call
130 time_left = self.throttle_period - time_since_last_call
130 time_left = self.throttle_period - time_since_last_call
131
131
132 if time_left > datetime.timedelta(seconds=0):
132 if time_left > datetime.timedelta(seconds=0):
133 return
133 return
134
134
135 self.time_of_last_call = datetime.datetime.now()
135 self.time_of_last_call = datetime.datetime.now()
136 return fn(*args, **kwargs)
136 return fn(*args, **kwargs)
137
137
138 return wrapper
138 return wrapper
139
139
140 def apply_throttle(value):
140 def apply_throttle(value):
141
141
142 @Throttle(seconds=value)
142 @Throttle(seconds=value)
143 def fnThrottled(fn):
143 def fnThrottled(fn):
144 fn()
144 fn()
145
145
146 return fnThrottled
146 return fnThrottled
147
147
148
148
149 @MPDecorator
149 @MPDecorator
150 class Plot(Operation):
150 class Plot(Operation):
151 """Base class for Schain plotting operations
151 """Base class for Schain plotting operations
152
152
153 This class should never be use directtly you must subclass a new operation,
153 This class should never be use directtly you must subclass a new operation,
154 children classes must be defined as follow:
154 children classes must be defined as follow:
155
155
156 ExamplePlot(Plot):
156 ExamplePlot(Plot):
157
157
158 CODE = 'code'
158 CODE = 'code'
159 colormap = 'jet'
159 colormap = 'jet'
160 plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
160 plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
161
161
162 def setup(self):
162 def setup(self):
163 pass
163 pass
164
164
165 def plot(self):
165 def plot(self):
166 pass
166 pass
167
167
168 """
168 """
169
169
170 CODE = 'Figure'
170 CODE = 'Figure'
171 colormap = 'jet'
171 colormap = 'jet'
172 bgcolor = 'white'
172 bgcolor = 'white'
173 buffering = True
173 buffering = True
174 __missing = 1E30
174 __missing = 1E30
175
175
176 __attrs__ = ['show', 'save', 'ymin', 'ymax', 'zmin', 'zmax', 'title',
176 __attrs__ = ['show', 'save', 'ymin', 'ymax', 'zmin', 'zmax', 'title',
177 'showprofile']
177 'showprofile']
178
178
179 def __init__(self):
179 def __init__(self):
180
180
181 Operation.__init__(self)
181 Operation.__init__(self)
182 self.isConfig = False
182 self.isConfig = False
183 self.isPlotConfig = False
183 self.isPlotConfig = False
184 self.save_time = 0
184 self.save_time = 0
185 self.sender_time = 0
185 self.sender_time = 0
186 self.data = None
186 self.data = None
187 self.firsttime = True
187 self.firsttime = True
188 self.sender_queue = deque(maxlen=10)
188 self.sender_queue = deque(maxlen=10)
189 self.plots_adjust = {'left': 0.125, 'right': 0.9, 'bottom': 0.15, 'top': 0.9, 'wspace': 0.2, 'hspace': 0.2}
189 self.plots_adjust = {'left': 0.125, 'right': 0.9, 'bottom': 0.15, 'top': 0.9, 'wspace': 0.2, 'hspace': 0.2}
190
190
191 def __fmtTime(self, x, pos):
191 def __fmtTime(self, x, pos):
192 '''
192 '''
193 '''
193 '''
194 if self.t_units == "h_m":
194 if self.t_units == "h_m":
195 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
195 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
196 if self.t_units == "h":
196 if self.t_units == "h":
197 return '{}'.format(self.getDateTime(x).strftime('%H'))
197 return '{}'.format(self.getDateTime(x).strftime('%H'))
198
198
199 def __setup(self, **kwargs):
199 def __setup(self, **kwargs):
200 '''
200 '''
201 Initialize variables
201 Initialize variables
202 '''
202 '''
203
203
204 self.figures = []
204 self.figures = []
205 self.axes = []
205 self.axes = []
206 self.cb_axes = []
206 self.cb_axes = []
207 self.pf_axes = []
207 self.pf_axes = []
208 self.localtime = kwargs.pop('localtime', True)
208 self.localtime = kwargs.pop('localtime', True)
209 self.show = kwargs.get('show', True)
209 self.show = kwargs.get('show', True)
210 self.save = kwargs.get('save', False)
210 self.save = kwargs.get('save', False)
211 self.save_period = kwargs.get('save_period', 0)
211 self.save_period = kwargs.get('save_period', 0)
212 self.colormap = kwargs.get('colormap', self.colormap)
212 self.colormap = kwargs.get('colormap', self.colormap)
213 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
213 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
214 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
214 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
215 self.colormaps = kwargs.get('colormaps', None)
215 self.colormaps = kwargs.get('colormaps', None)
216 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
216 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
217 self.showprofile = kwargs.get('showprofile', False)
217 self.showprofile = kwargs.get('showprofile', False)
218 self.title = kwargs.get('wintitle', self.CODE.upper())
218 self.title = kwargs.get('wintitle', self.CODE.upper())
219 self.cb_label = kwargs.get('cb_label', None)
219 self.cb_label = kwargs.get('cb_label', None)
220 self.cb_labels = kwargs.get('cb_labels', None)
220 self.cb_labels = kwargs.get('cb_labels', None)
221 self.labels = kwargs.get('labels', None)
221 self.labels = kwargs.get('labels', None)
222 self.xaxis = kwargs.get('xaxis', 'frequency')
222 self.xaxis = kwargs.get('xaxis', 'frequency')
223 self.zmin = kwargs.get('zmin', None)
223 self.zmin = kwargs.get('zmin', None)
224 self.zmax = kwargs.get('zmax', None)
224 self.zmax = kwargs.get('zmax', None)
225 self.zlimits = kwargs.get('zlimits', None)
225 self.zlimits = kwargs.get('zlimits', None)
226 self.xmin = kwargs.get('xmin', None)
226 self.xmin = kwargs.get('xmin', None)
227 self.xmax = kwargs.get('xmax', None)
227 self.xmax = kwargs.get('xmax', None)
228 self.xrange = kwargs.get('xrange', 12)
228 self.xrange = kwargs.get('xrange', 12)
229 self.xscale = kwargs.get('xscale', None)
229 self.xscale = kwargs.get('xscale', None)
230 self.ymin = kwargs.get('ymin', None)
230 self.ymin = kwargs.get('ymin', None)
231 self.ymax = kwargs.get('ymax', None)
231 self.ymax = kwargs.get('ymax', None)
232 self.yscale = kwargs.get('yscale', None)
232 self.yscale = kwargs.get('yscale', None)
233 self.xlabel = kwargs.get('xlabel', None)
233 self.xlabel = kwargs.get('xlabel', None)
234 self.attr_time = kwargs.get('attr_time', 'utctime')
234 self.attr_time = kwargs.get('attr_time', 'utctime')
235 self.attr_data = kwargs.get('attr_data', 'data_param')
235 self.attr_data = kwargs.get('attr_data', 'data_param')
236 self.decimation = kwargs.get('decimation', None)
236 self.decimation = kwargs.get('decimation', None)
237 self.oneFigure = kwargs.get('oneFigure', True)
237 self.oneFigure = kwargs.get('oneFigure', True)
238 self.width = kwargs.get('width', None)
238 self.width = kwargs.get('width', None)
239 self.height = kwargs.get('height', None)
239 self.height = kwargs.get('height', None)
240 self.colorbar = kwargs.get('colorbar', True)
240 self.colorbar = kwargs.get('colorbar', True)
241 self.factors = kwargs.get('factors', range(18))
241 self.factors = kwargs.get('factors', range(18))
242 self.channels = kwargs.get('channels', None)
242 self.channels = kwargs.get('channels', None)
243 self.titles = kwargs.get('titles', [])
243 self.titles = kwargs.get('titles', [])
244 self.polar = False
244 self.polar = False
245 self.type = kwargs.get('type', 'iq')
245 self.type = kwargs.get('type', 'iq')
246 self.grid = kwargs.get('grid', False)
246 self.grid = kwargs.get('grid', False)
247 self.pause = kwargs.get('pause', False)
247 self.pause = kwargs.get('pause', False)
248 self.save_code = kwargs.get('save_code', self.CODE)
248 self.save_code = kwargs.get('save_code', self.CODE)
249 self.throttle = kwargs.get('throttle', 0)
249 self.throttle = kwargs.get('throttle', 0)
250 self.exp_code = kwargs.get('exp_code', None)
250 self.exp_code = kwargs.get('exp_code', None)
251 self.server = kwargs.get('server', False)
251 self.server = kwargs.get('server', False)
252 self.sender_period = kwargs.get('sender_period', 60)
252 self.sender_period = kwargs.get('sender_period', 60)
253 self.tag = kwargs.get('tag', '')
253 self.tag = kwargs.get('tag', '')
254 self.height_index = kwargs.get('height_index', None)
254 self.height_index = kwargs.get('height_index', None)
255 self.__throttle_plot = apply_throttle(self.throttle)
255 self.__throttle_plot = apply_throttle(self.throttle)
256 code = self.attr_data if self.attr_data else self.CODE
256 code = self.attr_data if self.attr_data else self.CODE
257 self.data = PlotterData(self.CODE, self.exp_code, self.localtime)
257 self.data = PlotterData(self.CODE, self.exp_code, self.localtime)
258 self.tmin = kwargs.get('tmin', None)
258 self.tmin = kwargs.get('tmin', None)
259 self.t_units = kwargs.get('t_units', "h_m")
259 self.t_units = kwargs.get('t_units', "h_m")
260 self.selectedHeight = kwargs.get('selectedHeight', None)
261
260
262
261 if self.server:
263 if self.server:
262 if not self.server.startswith('tcp://'):
264 if not self.server.startswith('tcp://'):
263 self.server = 'tcp://{}'.format(self.server)
265 self.server = 'tcp://{}'.format(self.server)
264 log.success(
266 log.success(
265 'Sending to server: {}'.format(self.server),
267 'Sending to server: {}'.format(self.server),
266 self.name
268 self.name
267 )
269 )
268
270
269 if isinstance(self.attr_data, str):
271 if isinstance(self.attr_data, str):
270 self.attr_data = [self.attr_data]
272 self.attr_data = [self.attr_data]
271
273
272 def __setup_plot(self):
274 def __setup_plot(self):
273 '''
275 '''
274 Common setup for all figures, here figures and axes are created
276 Common setup for all figures, here figures and axes are created
275 '''
277 '''
276
278
277 self.setup()
279 self.setup()
278
280
279 self.time_label = 'LT' if self.localtime else 'UTC'
281 self.time_label = 'LT' if self.localtime else 'UTC'
280
282
281 if self.width is None:
283 if self.width is None:
282 self.width = 8
284 self.width = 8
283
285
284 self.figures = []
286 self.figures = []
285 self.axes = []
287 self.axes = []
286 self.cb_axes = []
288 self.cb_axes = []
287 self.pf_axes = []
289 self.pf_axes = []
288 self.cmaps = []
290 self.cmaps = []
289
291
290 size = '15%' if self.ncols == 1 else '30%'
292 size = '15%' if self.ncols == 1 else '30%'
291 pad = '4%' if self.ncols == 1 else '8%'
293 pad = '4%' if self.ncols == 1 else '8%'
292
294
293 if self.oneFigure:
295 if self.oneFigure:
294 if self.height is None:
296 if self.height is None:
295 self.height = 1.4 * self.nrows + 1
297 self.height = 1.4 * self.nrows + 1
296 fig = plt.figure(figsize=(self.width, self.height),
298 fig = plt.figure(figsize=(self.width, self.height),
297 edgecolor='k',
299 edgecolor='k',
298 facecolor='w')
300 facecolor='w')
299 self.figures.append(fig)
301 self.figures.append(fig)
300 for n in range(self.nplots):
302 for n in range(self.nplots):
301 ax = fig.add_subplot(self.nrows, self.ncols,
303 ax = fig.add_subplot(self.nrows, self.ncols,
302 n + 1, polar=self.polar)
304 n + 1, polar=self.polar)
303 ax.tick_params(labelsize=8)
305 ax.tick_params(labelsize=8)
304 ax.firsttime = True
306 ax.firsttime = True
305 ax.index = 0
307 ax.index = 0
306 ax.press = None
308 ax.press = None
307 self.axes.append(ax)
309 self.axes.append(ax)
308 if self.showprofile:
310 if self.showprofile:
309 cax = self.__add_axes(ax, size=size, pad=pad)
311 cax = self.__add_axes(ax, size=size, pad=pad)
310 cax.tick_params(labelsize=8)
312 cax.tick_params(labelsize=8)
311 self.pf_axes.append(cax)
313 self.pf_axes.append(cax)
312 else:
314 else:
313 if self.height is None:
315 if self.height is None:
314 self.height = 3
316 self.height = 3
315 for n in range(self.nplots):
317 for n in range(self.nplots):
316 fig = plt.figure(figsize=(self.width, self.height),
318 fig = plt.figure(figsize=(self.width, self.height),
317 edgecolor='k',
319 edgecolor='k',
318 facecolor='w')
320 facecolor='w')
319 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
321 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
320 ax.tick_params(labelsize=8)
322 ax.tick_params(labelsize=8)
321 ax.firsttime = True
323 ax.firsttime = True
322 ax.index = 0
324 ax.index = 0
323 ax.press = None
325 ax.press = None
324 self.figures.append(fig)
326 self.figures.append(fig)
325 self.axes.append(ax)
327 self.axes.append(ax)
326 if self.showprofile:
328 if self.showprofile:
327 cax = self.__add_axes(ax, size=size, pad=pad)
329 cax = self.__add_axes(ax, size=size, pad=pad)
328 cax.tick_params(labelsize=8)
330 cax.tick_params(labelsize=8)
329 self.pf_axes.append(cax)
331 self.pf_axes.append(cax)
330
332
331 for n in range(self.nrows):
333 for n in range(self.nrows):
332 if self.colormaps is not None:
334 if self.colormaps is not None:
333 cmap = plt.get_cmap(self.colormaps[n])
335 cmap = plt.get_cmap(self.colormaps[n])
334 else:
336 else:
335 cmap = plt.get_cmap(self.colormap)
337 cmap = plt.get_cmap(self.colormap)
336 cmap.set_bad(self.bgcolor, 1.)
338 cmap.set_bad(self.bgcolor, 1.)
337 self.cmaps.append(cmap)
339 self.cmaps.append(cmap)
338
340
339 def __add_axes(self, ax, size='30%', pad='8%'):
341 def __add_axes(self, ax, size='30%', pad='8%'):
340 '''
342 '''
341 Add new axes to the given figure
343 Add new axes to the given figure
342 '''
344 '''
343 divider = make_axes_locatable(ax)
345 divider = make_axes_locatable(ax)
344 nax = divider.new_horizontal(size=size, pad=pad)
346 nax = divider.new_horizontal(size=size, pad=pad)
345 ax.figure.add_axes(nax)
347 ax.figure.add_axes(nax)
346 return nax
348 return nax
347
349
348 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
350 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
349 '''
351 '''
350 Create a masked array for missing data
352 Create a masked array for missing data
351 '''
353 '''
352 if x_buffer.shape[0] < 2:
354 if x_buffer.shape[0] < 2:
353 return x_buffer, y_buffer, z_buffer
355 return x_buffer, y_buffer, z_buffer
354
356
355 deltas = x_buffer[1:] - x_buffer[0:-1]
357 deltas = x_buffer[1:] - x_buffer[0:-1]
356 x_median = numpy.median(deltas)
358 x_median = numpy.median(deltas)
357
359
358 index = numpy.where(deltas > 5 * x_median)
360 index = numpy.where(deltas > 5 * x_median)
359
361
360 if len(index[0]) != 0:
362 if len(index[0]) != 0:
361 z_buffer[::, index[0], ::] = self.__missing
363 z_buffer[::, index[0], ::] = self.__missing
362 z_buffer = numpy.ma.masked_inside(z_buffer,
364 z_buffer = numpy.ma.masked_inside(z_buffer,
363 0.99 * self.__missing,
365 0.99 * self.__missing,
364 1.01 * self.__missing)
366 1.01 * self.__missing)
365
367
366 return x_buffer, y_buffer, z_buffer
368 return x_buffer, y_buffer, z_buffer
367
369
368 def decimate(self):
370 def decimate(self):
369
371
370 # dx = int(len(self.x)/self.__MAXNUMX) + 1
372 # dx = int(len(self.x)/self.__MAXNUMX) + 1
371 dy = int(len(self.y) / self.decimation) + 1
373 dy = int(len(self.y) / self.decimation) + 1
372
374
373 # x = self.x[::dx]
375 # x = self.x[::dx]
374 x = self.x
376 x = self.x
375 y = self.y[::dy]
377 y = self.y[::dy]
376 z = self.z[::, ::, ::dy]
378 z = self.z[::, ::, ::dy]
377
379
378 return x, y, z
380 return x, y, z
379
381
380 def format(self):
382 def format(self):
381 '''
383 '''
382 Set min and max values, labels, ticks and titles
384 Set min and max values, labels, ticks and titles
383 '''
385 '''
384
386
385 for n, ax in enumerate(self.axes):
387 for n, ax in enumerate(self.axes):
386 if ax.firsttime:
388 if ax.firsttime:
387 if self.xaxis != 'time':
389 if self.xaxis != 'time':
388 xmin = self.xmin
390 xmin = self.xmin
389 xmax = self.xmax
391 xmax = self.xmax
390 else:
392 else:
391 xmin = self.tmin
393 xmin = self.tmin
392 xmax = self.tmin + self.xrange*60*60
394 xmax = self.tmin + self.xrange*60*60
393 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
395 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
394 if self.t_units == "h_m":
396 if self.t_units == "h_m":
395 ax.xaxis.set_major_locator(LinearLocator(9))
397 ax.xaxis.set_major_locator(LinearLocator(9))
396 if self.t_units == "h":
398 if self.t_units == "h":
397 ax.xaxis.set_major_locator(LinearLocator(int((xmax-xmin)/3600)+1))
399 ax.xaxis.set_major_locator(LinearLocator(int((xmax-xmin)/3600)+1))
398 ymin = self.ymin if self.ymin is not None else numpy.nanmin(self.y[numpy.isfinite(self.y)])
400 ymin = self.ymin if self.ymin is not None else numpy.nanmin(self.y[numpy.isfinite(self.y)])
399 ymax = self.ymax if self.ymax is not None else numpy.nanmax(self.y[numpy.isfinite(self.y)])
401 ymax = self.ymax if self.ymax is not None else numpy.nanmax(self.y[numpy.isfinite(self.y)])
400 ax.set_facecolor(self.bgcolor)
402 ax.set_facecolor(self.bgcolor)
401 if self.xscale:
403 if self.xscale:
402 ax.xaxis.set_major_formatter(FuncFormatter(
404 ax.xaxis.set_major_formatter(FuncFormatter(
403 lambda x, pos: '{0:g}'.format(x*self.xscale)))
405 lambda x, pos: '{0:g}'.format(x*self.xscale)))
404 if self.yscale:
406 if self.yscale:
405 ax.yaxis.set_major_formatter(FuncFormatter(
407 ax.yaxis.set_major_formatter(FuncFormatter(
406 lambda x, pos: '{0:g}'.format(x*self.yscale)))
408 lambda x, pos: '{0:g}'.format(x*self.yscale)))
407 if self.xlabel is not None:
409 if self.xlabel is not None:
408 ax.set_xlabel(self.xlabel)
410 ax.set_xlabel(self.xlabel)
409 if self.ylabel is not None:
411 if self.ylabel is not None:
410 ax.set_ylabel(self.ylabel)
412 ax.set_ylabel(self.ylabel)
411 if self.showprofile:
413 if self.showprofile:
412 self.pf_axes[n].set_ylim(ymin, ymax)
414 self.pf_axes[n].set_ylim(ymin, ymax)
413 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
415 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
414 self.pf_axes[n].set_xlabel('dB')
416 self.pf_axes[n].set_xlabel('dB')
415 self.pf_axes[n].grid(b=True, axis='x')
417 self.pf_axes[n].grid(b=True, axis='x')
416 [tick.set_visible(False)
418 [tick.set_visible(False)
417 for tick in self.pf_axes[n].get_yticklabels()]
419 for tick in self.pf_axes[n].get_yticklabels()]
418 if self.colorbar:
420 if self.colorbar:
419 ax.cbar = plt.colorbar(
421 ax.cbar = plt.colorbar(
420 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
422 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
421 ax.cbar.ax.tick_params(labelsize=8)
423 ax.cbar.ax.tick_params(labelsize=8)
422 ax.cbar.ax.press = None
424 ax.cbar.ax.press = None
423 if self.cb_label:
425 if self.cb_label:
424 ax.cbar.set_label(self.cb_label, size=8)
426 ax.cbar.set_label(self.cb_label, size=8)
425 elif self.cb_labels:
427 elif self.cb_labels:
426 ax.cbar.set_label(self.cb_labels[n], size=8)
428 ax.cbar.set_label(self.cb_labels[n], size=8)
427 else:
429 else:
428 ax.cbar = None
430 ax.cbar = None
429 ax.set_xlim(xmin, xmax)
431 ax.set_xlim(xmin, xmax)
430 ax.set_ylim(ymin, ymax)
432 ax.set_ylim(ymin, ymax)
431 ax.firsttime = False
433 ax.firsttime = False
432 if self.grid:
434 if self.grid:
433 ax.grid(True)
435 ax.grid(True)
434 if not self.polar:
436 if not self.polar:
435 ax.set_title('{} {} {}'.format(
437 ax.set_title('{} {} {}'.format(
436 self.titles[n],
438 self.titles[n],
437 self.getDateTime(self.data.max_time).strftime(
439 self.getDateTime(self.data.max_time).strftime(
438 '%Y-%m-%d %H:%M:%S'),
440 '%Y-%m-%d %H:%M:%S'),
439 self.time_label),
441 self.time_label),
440 size=8)
442 size=8)
441 else:
443 else:
442 ax.set_title('{}'.format(self.titles[n]), size=8)
444 ax.set_title('{}'.format(self.titles[n]), size=8)
443 ax.set_ylim(0, 90)
445 ax.set_ylim(0, 90)
444 ax.set_yticks(numpy.arange(0, 90, 20))
446 ax.set_yticks(numpy.arange(0, 90, 20))
445 ax.yaxis.labelpad = 40
447 ax.yaxis.labelpad = 40
446
448
447 if self.firsttime:
449 if self.firsttime:
448 for n, fig in enumerate(self.figures):
450 for n, fig in enumerate(self.figures):
449 fig.subplots_adjust(**self.plots_adjust)
451 fig.subplots_adjust(**self.plots_adjust)
450 self.firsttime = False
452 self.firsttime = False
451
453
452 def clear_figures(self):
454 def clear_figures(self):
453 '''
455 '''
454 Reset axes for redraw plots
456 Reset axes for redraw plots
455 '''
457 '''
456
458
457 for ax in self.axes+self.pf_axes+self.cb_axes:
459 for ax in self.axes+self.pf_axes+self.cb_axes:
458 ax.clear()
460 ax.clear()
459 ax.firsttime = True
461 ax.firsttime = True
460 if hasattr(ax, 'cbar') and ax.cbar:
462 if hasattr(ax, 'cbar') and ax.cbar:
461 ax.cbar.remove()
463 ax.cbar.remove()
462
464
463 def __plot(self):
465 def __plot(self):
464 '''
466 '''
465 Main function to plot, format and save figures
467 Main function to plot, format and save figures
466 '''
468 '''
467
469
468 self.plot()
470 self.plot()
469 self.format()
471 self.format()
470
472
471 for n, fig in enumerate(self.figures):
473 for n, fig in enumerate(self.figures):
472 if self.nrows == 0 or self.nplots == 0:
474 if self.nrows == 0 or self.nplots == 0:
473 log.warning('No data', self.name)
475 log.warning('No data', self.name)
474 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
476 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
475 fig.canvas.manager.set_window_title(self.CODE)
477 fig.canvas.manager.set_window_title(self.CODE)
476 continue
478 continue
477
479
478 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
480 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
479 self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
481 self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
480 fig.canvas.draw()
482 fig.canvas.draw()
481 if self.show:
483 if self.show:
482 fig.show()
484 fig.show()
483 figpause(0.01)
485 figpause(0.01)
484
486
485 if self.save:
487 if self.save:
486 self.save_figure(n)
488 self.save_figure(n)
487
489
488 if self.server:
490 if self.server:
489 self.send_to_server()
491 self.send_to_server()
490
492
491 def __update(self, dataOut, timestamp):
493 def __update(self, dataOut, timestamp):
492 '''
494 '''
493 '''
495 '''
494
496
495 metadata = {
497 metadata = {
496 'yrange': dataOut.heightList,
498 'yrange': dataOut.heightList,
497 'interval': dataOut.timeInterval,
499 'interval': dataOut.timeInterval,
498 'channels': dataOut.channelList
500 'channels': dataOut.channelList
499 }
501 }
500 data, meta = self.update(dataOut)
502 data, meta = self.update(dataOut)
501 metadata.update(meta)
503 metadata.update(meta)
502 self.data.update(data, timestamp, metadata)
504 self.data.update(data, timestamp, metadata)
503
505
504 def save_figure(self, n):
506 def save_figure(self, n):
505 '''
507 '''
506 '''
508 '''
507
509
508 if (self.data.max_time - self.save_time) <= self.save_period:
510 if (self.data.max_time - self.save_time) <= self.save_period:
509 return
511 return
510
512
511 self.save_time = self.data.max_time
513 self.save_time = self.data.max_time
512
514
513 fig = self.figures[n]
515 fig = self.figures[n]
514
516
515 if self.throttle == 0:
517 if self.throttle == 0:
516 figname = os.path.join(
518 figname = os.path.join(
517 self.save,
519 self.save,
518 self.save_code,
520 self.save_code,
519 '{}_{}.png'.format(
521 '{}_{}.png'.format(
520 self.save_code,
522 self.save_code,
521 self.getDateTime(self.data.max_time).strftime(
523 self.getDateTime(self.data.max_time).strftime(
522 '%Y%m%d_%H%M%S'
524 '%Y%m%d_%H%M%S'
523 ),
525 ),
524 )
526 )
525 )
527 )
526 log.log('Saving figure: {}'.format(figname), self.name)
528 log.log('Saving figure: {}'.format(figname), self.name)
527 if not os.path.isdir(os.path.dirname(figname)):
529 if not os.path.isdir(os.path.dirname(figname)):
528 os.makedirs(os.path.dirname(figname))
530 os.makedirs(os.path.dirname(figname))
529 fig.savefig(figname)
531 fig.savefig(figname)
530
532
531 figname = os.path.join(
533 figname = os.path.join(
532 self.save,
534 self.save,
533 '{}_{}.png'.format(
535 '{}_{}.png'.format(
534 self.save_code,
536 self.save_code,
535 self.getDateTime(self.data.min_time).strftime(
537 self.getDateTime(self.data.min_time).strftime(
536 '%Y%m%d'
538 '%Y%m%d'
537 ),
539 ),
538 )
540 )
539 )
541 )
540
542
541 log.log('Saving figure: {}'.format(figname), self.name)
543 log.log('Saving figure: {}'.format(figname), self.name)
542 if not os.path.isdir(os.path.dirname(figname)):
544 if not os.path.isdir(os.path.dirname(figname)):
543 os.makedirs(os.path.dirname(figname))
545 os.makedirs(os.path.dirname(figname))
544 fig.savefig(figname)
546 fig.savefig(figname)
545
547
546 def send_to_server(self):
548 def send_to_server(self):
547 '''
549 '''
548 '''
550 '''
549
551
550 if self.exp_code == None:
552 if self.exp_code == None:
551 log.warning('Missing `exp_code` skipping sending to server...')
553 log.warning('Missing `exp_code` skipping sending to server...')
552
554
553 last_time = self.data.max_time
555 last_time = self.data.max_time
554 interval = last_time - self.sender_time
556 interval = last_time - self.sender_time
555 if interval < self.sender_period:
557 if interval < self.sender_period:
556 return
558 return
557
559
558 self.sender_time = last_time
560 self.sender_time = last_time
559
561
560 attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax']
562 attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax']
561 for attr in attrs:
563 for attr in attrs:
562 value = getattr(self, attr)
564 value = getattr(self, attr)
563 if value:
565 if value:
564 if isinstance(value, (numpy.float32, numpy.float64)):
566 if isinstance(value, (numpy.float32, numpy.float64)):
565 value = round(float(value), 2)
567 value = round(float(value), 2)
566 self.data.meta[attr] = value
568 self.data.meta[attr] = value
567 if self.colormap == 'jet':
569 if self.colormap == 'jet':
568 self.data.meta['colormap'] = 'Jet'
570 self.data.meta['colormap'] = 'Jet'
569 elif 'RdBu' in self.colormap:
571 elif 'RdBu' in self.colormap:
570 self.data.meta['colormap'] = 'RdBu'
572 self.data.meta['colormap'] = 'RdBu'
571 else:
573 else:
572 self.data.meta['colormap'] = 'Viridis'
574 self.data.meta['colormap'] = 'Viridis'
573 self.data.meta['interval'] = int(interval)
575 self.data.meta['interval'] = int(interval)
574
576
575 self.sender_queue.append(last_time)
577 self.sender_queue.append(last_time)
576
578
577 while True:
579 while True:
578 try:
580 try:
579 tm = self.sender_queue.popleft()
581 tm = self.sender_queue.popleft()
580 except IndexError:
582 except IndexError:
581 break
583 break
582 msg = self.data.jsonify(tm, self.save_code, self.plot_type)
584 msg = self.data.jsonify(tm, self.save_code, self.plot_type)
583 self.socket.send_string(msg)
585 self.socket.send_string(msg)
584 socks = dict(self.poll.poll(2000))
586 socks = dict(self.poll.poll(2000))
585 if socks.get(self.socket) == zmq.POLLIN:
587 if socks.get(self.socket) == zmq.POLLIN:
586 reply = self.socket.recv_string()
588 reply = self.socket.recv_string()
587 if reply == 'ok':
589 if reply == 'ok':
588 log.log("Response from server ok", self.name)
590 log.log("Response from server ok", self.name)
589 time.sleep(0.1)
591 time.sleep(0.1)
590 continue
592 continue
591 else:
593 else:
592 log.warning(
594 log.warning(
593 "Malformed reply from server: {}".format(reply), self.name)
595 "Malformed reply from server: {}".format(reply), self.name)
594 else:
596 else:
595 log.warning(
597 log.warning(
596 "No response from server, retrying...", self.name)
598 "No response from server, retrying...", self.name)
597 self.sender_queue.appendleft(tm)
599 self.sender_queue.appendleft(tm)
598 self.socket.setsockopt(zmq.LINGER, 0)
600 self.socket.setsockopt(zmq.LINGER, 0)
599 self.socket.close()
601 self.socket.close()
600 self.poll.unregister(self.socket)
602 self.poll.unregister(self.socket)
601 self.socket = self.context.socket(zmq.REQ)
603 self.socket = self.context.socket(zmq.REQ)
602 self.socket.connect(self.server)
604 self.socket.connect(self.server)
603 self.poll.register(self.socket, zmq.POLLIN)
605 self.poll.register(self.socket, zmq.POLLIN)
604 break
606 break
605
607
606 def setup(self):
608 def setup(self):
607 '''
609 '''
608 This method should be implemented in the child class, the following
610 This method should be implemented in the child class, the following
609 attributes should be set:
611 attributes should be set:
610
612
611 self.nrows: number of rows
613 self.nrows: number of rows
612 self.ncols: number of cols
614 self.ncols: number of cols
613 self.nplots: number of plots (channels or pairs)
615 self.nplots: number of plots (channels or pairs)
614 self.ylabel: label for Y axes
616 self.ylabel: label for Y axes
615 self.titles: list of axes title
617 self.titles: list of axes title
616
618
617 '''
619 '''
618 raise NotImplementedError
620 raise NotImplementedError
619
621
620 def plot(self):
622 def plot(self):
621 '''
623 '''
622 Must be defined in the child class, the actual plotting method
624 Must be defined in the child class, the actual plotting method
623 '''
625 '''
624 raise NotImplementedError
626 raise NotImplementedError
625
627
626 def update(self, dataOut):
628 def update(self, dataOut):
627 '''
629 '''
628 Must be defined in the child class, update self.data with new data
630 Must be defined in the child class, update self.data with new data
629 '''
631 '''
630
632
631 data = {
633 data = {
632 self.CODE: getattr(dataOut, 'data_{}'.format(self.CODE))
634 self.CODE: getattr(dataOut, 'data_{}'.format(self.CODE))
633 }
635 }
634 meta = {}
636 meta = {}
635
637
636 return data, meta
638 return data, meta
637
639
638 def run(self, dataOut, **kwargs):
640 def run(self, dataOut, **kwargs):
639 '''
641 '''
640 Main plotting routine
642 Main plotting routine
641 '''
643 '''
642 if self.isConfig is False:
644 if self.isConfig is False:
643 self.__setup(**kwargs)
645 self.__setup(**kwargs)
644
646
645 if self.localtime:
647 if self.localtime:
646 self.getDateTime = datetime.datetime.fromtimestamp
648 self.getDateTime = datetime.datetime.fromtimestamp
647 else:
649 else:
648 self.getDateTime = datetime.datetime.utcfromtimestamp
650 self.getDateTime = datetime.datetime.utcfromtimestamp
649
651
650 self.data.setup()
652 self.data.setup()
651 self.isConfig = True
653 self.isConfig = True
652 if self.server:
654 if self.server:
653 self.context = zmq.Context()
655 self.context = zmq.Context()
654 self.socket = self.context.socket(zmq.REQ)
656 self.socket = self.context.socket(zmq.REQ)
655 self.socket.connect(self.server)
657 self.socket.connect(self.server)
656 self.poll = zmq.Poller()
658 self.poll = zmq.Poller()
657 self.poll.register(self.socket, zmq.POLLIN)
659 self.poll.register(self.socket, zmq.POLLIN)
658
660
659 tm = getattr(dataOut, self.attr_time)
661 tm = getattr(dataOut, self.attr_time)
660
662
661 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
663 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
662 self.save_time = tm
664 self.save_time = tm
663 self.__plot()
665 self.__plot()
664 self.tmin += self.xrange*60*60
666 self.tmin += self.xrange*60*60
665 self.data.setup()
667 self.data.setup()
666 self.clear_figures()
668 self.clear_figures()
667
669
668 self.__update(dataOut, tm)
670 self.__update(dataOut, tm)
669
671
670 if self.isPlotConfig is False:
672 if self.isPlotConfig is False:
671 self.__setup_plot()
673 self.__setup_plot()
672 self.isPlotConfig = True
674 self.isPlotConfig = True
673 if self.xaxis == 'time':
675 if self.xaxis == 'time':
674 dt = self.getDateTime(tm)
676 dt = self.getDateTime(tm)
675 if self.xmin is None:
677 if self.xmin is None:
676 self.tmin = tm
678 self.tmin = tm
677 self.xmin = dt.hour
679 self.xmin = dt.hour
678 minutes = (self.xmin-int(self.xmin)) * 60
680 minutes = (self.xmin-int(self.xmin)) * 60
679 seconds = (minutes - int(minutes)) * 60
681 seconds = (minutes - int(minutes)) * 60
680 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
682 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
681 datetime.datetime(1970, 1, 1)).total_seconds()
683 datetime.datetime(1970, 1, 1)).total_seconds()
682 if self.localtime:
684 if self.localtime:
683 self.tmin += time.timezone
685 self.tmin += time.timezone
684
686
685 if self.xmin is not None and self.xmax is not None:
687 if self.xmin is not None and self.xmax is not None:
686 self.xrange = self.xmax - self.xmin
688 self.xrange = self.xmax - self.xmin
687
689
688 if self.throttle == 0:
690 if self.throttle == 0:
689 self.__plot()
691 self.__plot()
690 else:
692 else:
691 self.__throttle_plot(self.__plot)#, coerce=coerce)
693 self.__throttle_plot(self.__plot)#, coerce=coerce)
692
694
693 def close(self):
695 def close(self):
694
696
695 if self.data and not self.data.flagNoData:
697 if self.data and not self.data.flagNoData:
696 self.save_time = 0
698 self.save_time = 0
697 self.__plot()
699 self.__plot()
698 if self.data and not self.data.flagNoData and self.pause:
700 if self.data and not self.data.flagNoData and self.pause:
699 figpause(10)
701 figpause(10)
Show file before | Show file after | Hide comments
@@ -1,102 +1,107
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """Classes to plo Specra Heis data
5 """Classes to plo Specra Heis data
6
6
7 """
7 """
8
8
9 import numpy
9 import numpy
10
10
11 from schainpy.model.graphics.jroplot_base import Plot, plt
11 from schainpy.model.graphics.jroplot_base import Plot, plt
12 import matplotlib.pyplot as plt
12 import matplotlib.pyplot as plt
13
13
14 class SpectraHeisPlot(Plot):
14 class SpectraHeisPlot(Plot):
15
15
16 CODE = 'spc_heis'
16 CODE = 'spc_heis'
17
17 channelList = []
18 def setup(self):
18 def setup(self):
19
19
20 self.nplots = len(self.data.channels)
20 self.nplots = len(self.data.channels)
21 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
21 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
22 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
22 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
23 self.height = 2.6 * self.nrows
23 self.height = 2.6 * self.nrows
24 self.width = 3.5 * self.ncols
24 self.width = 3.5 * self.ncols
25 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.95, 'bottom': 0.08})
25 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.95, 'bottom': 0.08})
26 self.ylabel = 'Intensity [dB]'
26 self.ylabel = 'Intensity [dB]'
27 self.xlabel = 'Frequency [KHz]'
27 self.xlabel = 'Frequency [KHz]'
28 self.colorbar = False
28 self.colorbar = False
29
29
30 def update(self, dataOut):
30 def update(self, dataOut):
31 if len(self.channelList) == 0:
32 self.channelList = dataOut.channelList
31
33
32 data = {}
34 data = {}
33 meta = {}
35 meta = {}
34 spc = 10*numpy.log10(dataOut.data_spc / dataOut.normFactor)
36 spc = 10*numpy.log10(dataOut.data_spc / dataOut.normFactor)
35 data['spc_heis'] = spc
37 data['spc_heis'] = spc
36
38
37 return data, meta
39 return data, meta
38
40
39 def plot(self):
41 def plot(self):
40
42
41 c = 3E8
43 c = 3E8
42 deltaHeight = self.data.yrange[1] - self.data.yrange[0] # yrange = heightList
44 deltaHeight = self.data.yrange[1] - self.data.yrange[0] # yrange = heightList
43 x = numpy.arange(-1*len(self.data.yrange)/2., len(self.data.yrange)/2.)*(c/(2*deltaHeight*len(self.data.yrange)*1000))
45 x = numpy.arange(-1*len(self.data.yrange)/2., len(self.data.yrange)/2.)*(c/(2*deltaHeight*len(self.data.yrange)*1000))
44 self.y = self.data[-1]['spc_heis']
46 self.y = self.data[-1]['spc_heis']
45 self.titles = []
47 self.titles = []
46
48
47 Maintitle = "Range from %d km to %d km" %(int(self.data.yrange[0]),int(self.data.yrange[-1]))
49 Maintitle = "Range from %d km to %d km" %(int(self.data.yrange[0]),int(self.data.yrange[-1]))
48 for n, ax in enumerate(self.axes):
50 for n, ax in enumerate(self.axes):
49 ychannel = self.y[n,:]
51 ychannel = self.y[n,:]
50 if ax.firsttime:
52 if ax.firsttime:
51 self.xmin = min(x) if self.xmin is None else self.xmin
53 self.xmin = min(x) if self.xmin is None else self.xmin
52 self.xmax = max(x) if self.xmax is None else self.xmax
54 self.xmax = max(x) if self.xmax is None else self.xmax
53 ax.plt = ax.plot(x, ychannel, lw=1, color='b')[0]
55 ax.plt = ax.plot(x, ychannel, lw=1, color='b')[0]
56 ax.set_ylim(ymin=self.zmin, ymax=self.zmax)
57 ax.set_xlim(xmin=self.xmin, xmax=self.xmax)
54 else:
58 else:
55 ax.plt.set_data(x, ychannel)
59 ax.plt.set_data(x, ychannel)
56
60 ax.set_ylim(ymin=self.zmin, ymax=self.zmax)
61 ax.set_xlim(xmin=self.xmin, xmax=self.xmax)
57 self.titles.append("Channel {}: {:4.2f}dB".format(n, numpy.max(ychannel)))
62 self.titles.append("Channel {}: {:4.2f}dB".format(n, numpy.max(ychannel)))
58 plt.suptitle(Maintitle)
63 plt.suptitle(Maintitle)
59
64
60 class RTIHeisPlot(Plot):
65 class RTIHeisPlot(Plot):
61
66
62 CODE = 'rti_heis'
67 CODE = 'rti_heis'
63
68
64 def setup(self):
69 def setup(self):
65
70
66 self.xaxis = 'time'
71 self.xaxis = 'time'
67 self.ncols = 1
72 self.ncols = 1
68 self.nrows = 1
73 self.nrows = 1
69 self.nplots = 1
74 self.nplots = 1
70 self.ylabel = 'Intensity [dB]'
75 self.ylabel = 'Intensity [dB]'
71 self.xlabel = 'Time'
76 self.xlabel = 'Time'
72 self.titles = ['RTI']
77 self.titles = ['RTI']
73 self.colorbar = False
78 self.colorbar = False
74 self.height = 4
79 self.height = 4
75 self.plots_adjust.update({'right': 0.85 })
80 self.plots_adjust.update({'right': 0.85 })
76
81
77 def update(self, dataOut):
82 def update(self, dataOut):
78
83
79 data = {}
84 data = {}
80 meta = {}
85 meta = {}
81 spc = dataOut.data_spc / dataOut.normFactor
86 spc = dataOut.data_spc / dataOut.normFactor
82 spc = 10*numpy.log10(numpy.average(spc, axis=1))
87 spc = 10*numpy.log10(numpy.average(spc, axis=1))
83 data['rti_heis'] = spc
88 data['rti_heis'] = spc
84
89
85 return data, meta
90 return data, meta
86
91
87 def plot(self):
92 def plot(self):
88
93
89 x = self.data.times
94 x = self.data.times
90 Y = self.data['rti_heis']
95 Y = self.data['rti_heis']
91
96
92 if self.axes[0].firsttime:
97 if self.axes[0].firsttime:
93 self.ymin = numpy.nanmin(Y) - 5 if self.ymin == None else self.ymin
98 self.ymin = numpy.nanmin(Y) - 5 if self.ymin == None else self.ymin
94 self.ymax = numpy.nanmax(Y) + 5 if self.ymax == None else self.ymax
99 self.ymax = numpy.nanmax(Y) + 5 if self.ymax == None else self.ymax
95 for ch in self.data.channels:
100 for ch in self.data.channels:
96 y = Y[ch]
101 y = Y[ch]
97 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
102 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
98 plt.legend(bbox_to_anchor=(1.18, 1.0))
103 plt.legend(bbox_to_anchor=(1.18, 1.0))
99 else:
104 else:
100 for ch in self.data.channels:
105 for ch in self.data.channels:
101 y = Y[ch]
106 y = Y[ch]
102 self.axes[0].lines[ch].set_data(x, y)
107 self.axes[0].lines[ch].set_data(x, y)
Show file before | Show file after | Hide comments
@@ -1,355 +1,357
1 import os
1 import os
2 import datetime
2 import datetime
3 import numpy
3 import numpy
4
4
5 from schainpy.model.graphics.jroplot_base import Plot, plt
5 from schainpy.model.graphics.jroplot_base import Plot, plt
6 from schainpy.model.graphics.jroplot_spectra import SpectraPlot, RTIPlot, CoherencePlot
6 from schainpy.model.graphics.jroplot_spectra import SpectraPlot, RTIPlot, CoherencePlot
7 from schainpy.utils import log
7 from schainpy.utils import log
8
8
9 EARTH_RADIUS = 6.3710e3
9 EARTH_RADIUS = 6.3710e3
10
10
11
11
12 def ll2xy(lat1, lon1, lat2, lon2):
12 def ll2xy(lat1, lon1, lat2, lon2):
13
13
14 p = 0.017453292519943295
14 p = 0.017453292519943295
15 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
15 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
16 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
16 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
17 r = 12742 * numpy.arcsin(numpy.sqrt(a))
17 r = 12742 * numpy.arcsin(numpy.sqrt(a))
18 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
18 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
19 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
19 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
20 theta = -theta + numpy.pi/2
20 theta = -theta + numpy.pi/2
21 return r*numpy.cos(theta), r*numpy.sin(theta)
21 return r*numpy.cos(theta), r*numpy.sin(theta)
22
22
23
23
24 def km2deg(km):
24 def km2deg(km):
25 '''
25 '''
26 Convert distance in km to degrees
26 Convert distance in km to degrees
27 '''
27 '''
28
28
29 return numpy.rad2deg(km/EARTH_RADIUS)
29 return numpy.rad2deg(km/EARTH_RADIUS)
30
30
31
31
32
32
33 class SpectralMomentsPlot(SpectraPlot):
33 class SpectralMomentsPlot(SpectraPlot):
34 '''
34 '''
35 Plot for Spectral Moments
35 Plot for Spectral Moments
36 '''
36 '''
37 CODE = 'spc_moments'
37 CODE = 'spc_moments'
38 colormap = 'jet'
38 colormap = 'jet'
39 plot_type = 'pcolor'
39 plot_type = 'pcolor'
40
40
41
41
42 class SnrPlot(RTIPlot):
42 class SnrPlot(RTIPlot):
43 '''
43 '''
44 Plot for SNR Data
44 Plot for SNR Data
45 '''
45 '''
46
46
47 CODE = 'snr'
47 CODE = 'snr'
48 colormap = 'jet'
48 colormap = 'jet'
49
49
50 def update(self, dataOut):
50 def update(self, dataOut):
51 self.update_list(dataOut)
51 if len(self.channelList) == 0:
52 data = {
52 self.update_list(dataOut)
53 'snr': 10*numpy.log10(dataOut.data_snr)
53 data = {}
54 }
54 meta = {}
55
55 data['snr'] = 10*numpy.log10(dataOut.data_snr)
56 return data, {}
56
57 return data, meta
57
58
58 class DopplerPlot(RTIPlot):
59 class DopplerPlot(RTIPlot):
59 '''
60 '''
60 Plot for DOPPLER Data (1st moment)
61 Plot for DOPPLER Data (1st moment)
61 '''
62 '''
62
63
63 CODE = 'dop'
64 CODE = 'dop'
64 colormap = 'jet'
65 colormap = 'jet'
65
66
66 def update(self, dataOut):
67 def update(self, dataOut):
67 self.update_list(dataOut)
68 self.update_list(dataOut)
68 data = {
69 data = {
69 'dop': 10*numpy.log10(dataOut.data_dop)
70 'dop': 10*numpy.log10(dataOut.data_dop)
70 }
71 }
71
72
72 return data, {}
73 return data, {}
73
74
74 class PowerPlot(RTIPlot):
75 class PowerPlot(RTIPlot):
75 '''
76 '''
76 Plot for Power Data (0 moment)
77 Plot for Power Data (0 moment)
77 '''
78 '''
78
79
79 CODE = 'pow'
80 CODE = 'pow'
80 colormap = 'jet'
81 colormap = 'jet'
81
82
82 def update(self, dataOut):
83 def update(self, dataOut):
83 self.update_list(dataOut)
84 self.update_list(dataOut)
84 data = {
85 data = {
85 'pow': 10*numpy.log10(dataOut.data_pow)
86 'pow': 10*numpy.log10(dataOut.data_pow)
86 }
87 }
88 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
87 return data, {}
89 return data, {}
88
90
89 class SpectralWidthPlot(RTIPlot):
91 class SpectralWidthPlot(RTIPlot):
90 '''
92 '''
91 Plot for Spectral Width Data (2nd moment)
93 Plot for Spectral Width Data (2nd moment)
92 '''
94 '''
93
95
94 CODE = 'width'
96 CODE = 'width'
95 colormap = 'jet'
97 colormap = 'jet'
96
98
97 def update(self, dataOut):
99 def update(self, dataOut):
98 self.update_list(dataOut)
100 self.update_list(dataOut)
99 data = {
101 data = {
100 'width': dataOut.data_width
102 'width': dataOut.data_width
101 }
103 }
102
104 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
103 return data, {}
105 return data, {}
104
106
105 class SkyMapPlot(Plot):
107 class SkyMapPlot(Plot):
106 '''
108 '''
107 Plot for meteors detection data
109 Plot for meteors detection data
108 '''
110 '''
109
111
110 CODE = 'param'
112 CODE = 'param'
111
113
112 def setup(self):
114 def setup(self):
113
115
114 self.ncols = 1
116 self.ncols = 1
115 self.nrows = 1
117 self.nrows = 1
116 self.width = 7.2
118 self.width = 7.2
117 self.height = 7.2
119 self.height = 7.2
118 self.nplots = 1
120 self.nplots = 1
119 self.xlabel = 'Zonal Zenith Angle (deg)'
121 self.xlabel = 'Zonal Zenith Angle (deg)'
120 self.ylabel = 'Meridional Zenith Angle (deg)'
122 self.ylabel = 'Meridional Zenith Angle (deg)'
121 self.polar = True
123 self.polar = True
122 self.ymin = -180
124 self.ymin = -180
123 self.ymax = 180
125 self.ymax = 180
124 self.colorbar = False
126 self.colorbar = False
125
127
126 def plot(self):
128 def plot(self):
127
129
128 arrayParameters = numpy.concatenate(self.data['param'])
130 arrayParameters = numpy.concatenate(self.data['param'])
129 error = arrayParameters[:, -1]
131 error = arrayParameters[:, -1]
130 indValid = numpy.where(error == 0)[0]
132 indValid = numpy.where(error == 0)[0]
131 finalMeteor = arrayParameters[indValid, :]
133 finalMeteor = arrayParameters[indValid, :]
132 finalAzimuth = finalMeteor[:, 3]
134 finalAzimuth = finalMeteor[:, 3]
133 finalZenith = finalMeteor[:, 4]
135 finalZenith = finalMeteor[:, 4]
134
136
135 x = finalAzimuth * numpy.pi / 180
137 x = finalAzimuth * numpy.pi / 180
136 y = finalZenith
138 y = finalZenith
137
139
138 ax = self.axes[0]
140 ax = self.axes[0]
139
141
140 if ax.firsttime:
142 if ax.firsttime:
141 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
143 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
142 else:
144 else:
143 ax.plot.set_data(x, y)
145 ax.plot.set_data(x, y)
144
146
145 dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
147 dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
146 dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
148 dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
147 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
149 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
148 dt2,
150 dt2,
149 len(x))
151 len(x))
150 self.titles[0] = title
152 self.titles[0] = title
151
153
152
154
153 class GenericRTIPlot(Plot):
155 class GenericRTIPlot(Plot):
154 '''
156 '''
155 Plot for data_xxxx object
157 Plot for data_xxxx object
156 '''
158 '''
157
159
158 CODE = 'param'
160 CODE = 'param'
159 colormap = 'viridis'
161 colormap = 'viridis'
160 plot_type = 'pcolorbuffer'
162 plot_type = 'pcolorbuffer'
161
163
162 def setup(self):
164 def setup(self):
163 self.xaxis = 'time'
165 self.xaxis = 'time'
164 self.ncols = 1
166 self.ncols = 1
165 self.nrows = self.data.shape('param')[0]
167 self.nrows = self.data.shape('param')[0]
166 self.nplots = self.nrows
168 self.nplots = self.nrows
167 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
169 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
168
170
169 if not self.xlabel:
171 if not self.xlabel:
170 self.xlabel = 'Time'
172 self.xlabel = 'Time'
171
173
172 self.ylabel = 'Height [km]'
174 self.ylabel = 'Height [km]'
173 if not self.titles:
175 if not self.titles:
174 self.titles = ['Param {}'.format(x) for x in range(self.nrows)]
176 self.titles = ['Param {}'.format(x) for x in range(self.nrows)]
175
177
176 def update(self, dataOut):
178 def update(self, dataOut):
177
179
178 data = {
180 data = {
179 'param' : numpy.concatenate([getattr(dataOut, attr) for attr in self.attr_data], axis=0)
181 'param' : numpy.concatenate([getattr(dataOut, attr) for attr in self.attr_data], axis=0)
180 }
182 }
181
183
182 meta = {}
184 meta = {}
183
185
184 return data, meta
186 return data, meta
185
187
186 def plot(self):
188 def plot(self):
187 # self.data.normalize_heights()
189 # self.data.normalize_heights()
188 self.x = self.data.times
190 self.x = self.data.times
189 self.y = self.data.yrange
191 self.y = self.data.yrange
190 self.z = self.data['param']
192 self.z = self.data['param']
191
193
192 self.z = numpy.ma.masked_invalid(self.z)
194 self.z = numpy.ma.masked_invalid(self.z)
193
195
194 if self.decimation is None:
196 if self.decimation is None:
195 x, y, z = self.fill_gaps(self.x, self.y, self.z)
197 x, y, z = self.fill_gaps(self.x, self.y, self.z)
196 else:
198 else:
197 x, y, z = self.fill_gaps(*self.decimate())
199 x, y, z = self.fill_gaps(*self.decimate())
198
200
199 for n, ax in enumerate(self.axes):
201 for n, ax in enumerate(self.axes):
200
202
201 self.zmax = self.zmax if self.zmax is not None else numpy.max(
203 self.zmax = self.zmax if self.zmax is not None else numpy.max(
202 self.z[n])
204 self.z[n])
203 self.zmin = self.zmin if self.zmin is not None else numpy.min(
205 self.zmin = self.zmin if self.zmin is not None else numpy.min(
204 self.z[n])
206 self.z[n])
205
207
206 if ax.firsttime:
208 if ax.firsttime:
207 if self.zlimits is not None:
209 if self.zlimits is not None:
208 self.zmin, self.zmax = self.zlimits[n]
210 self.zmin, self.zmax = self.zlimits[n]
209
211
210 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
212 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
211 vmin=self.zmin,
213 vmin=self.zmin,
212 vmax=self.zmax,
214 vmax=self.zmax,
213 cmap=self.cmaps[n]
215 cmap=self.cmaps[n]
214 )
216 )
215 else:
217 else:
216 if self.zlimits is not None:
218 if self.zlimits is not None:
217 self.zmin, self.zmax = self.zlimits[n]
219 self.zmin, self.zmax = self.zlimits[n]
218 ax.collections.remove(ax.collections[0])
220 ax.collections.remove(ax.collections[0])
219 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
221 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
220 vmin=self.zmin,
222 vmin=self.zmin,
221 vmax=self.zmax,
223 vmax=self.zmax,
222 cmap=self.cmaps[n]
224 cmap=self.cmaps[n]
223 )
225 )
224
226
225
227
226 class PolarMapPlot(Plot):
228 class PolarMapPlot(Plot):
227 '''
229 '''
228 Plot for weather radar
230 Plot for weather radar
229 '''
231 '''
230
232
231 CODE = 'param'
233 CODE = 'param'
232 colormap = 'seismic'
234 colormap = 'seismic'
233
235
234 def setup(self):
236 def setup(self):
235 self.ncols = 1
237 self.ncols = 1
236 self.nrows = 1
238 self.nrows = 1
237 self.width = 9
239 self.width = 9
238 self.height = 8
240 self.height = 8
239 self.mode = self.data.meta['mode']
241 self.mode = self.data.meta['mode']
240 if self.channels is not None:
242 if self.channels is not None:
241 self.nplots = len(self.channels)
243 self.nplots = len(self.channels)
242 self.nrows = len(self.channels)
244 self.nrows = len(self.channels)
243 else:
245 else:
244 self.nplots = self.data.shape(self.CODE)[0]
246 self.nplots = self.data.shape(self.CODE)[0]
245 self.nrows = self.nplots
247 self.nrows = self.nplots
246 self.channels = list(range(self.nplots))
248 self.channels = list(range(self.nplots))
247 if self.mode == 'E':
249 if self.mode == 'E':
248 self.xlabel = 'Longitude'
250 self.xlabel = 'Longitude'
249 self.ylabel = 'Latitude'
251 self.ylabel = 'Latitude'
250 else:
252 else:
251 self.xlabel = 'Range (km)'
253 self.xlabel = 'Range (km)'
252 self.ylabel = 'Height (km)'
254 self.ylabel = 'Height (km)'
253 self.bgcolor = 'white'
255 self.bgcolor = 'white'
254 self.cb_labels = self.data.meta['units']
256 self.cb_labels = self.data.meta['units']
255 self.lat = self.data.meta['latitude']
257 self.lat = self.data.meta['latitude']
256 self.lon = self.data.meta['longitude']
258 self.lon = self.data.meta['longitude']
257 self.xmin, self.xmax = float(
259 self.xmin, self.xmax = float(
258 km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
260 km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
259 self.ymin, self.ymax = float(
261 self.ymin, self.ymax = float(
260 km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
262 km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
261 # self.polar = True
263 # self.polar = True
262
264
263 def plot(self):
265 def plot(self):
264
266
265 for n, ax in enumerate(self.axes):
267 for n, ax in enumerate(self.axes):
266 data = self.data['param'][self.channels[n]]
268 data = self.data['param'][self.channels[n]]
267
269
268 zeniths = numpy.linspace(
270 zeniths = numpy.linspace(
269 0, self.data.meta['max_range'], data.shape[1])
271 0, self.data.meta['max_range'], data.shape[1])
270 if self.mode == 'E':
272 if self.mode == 'E':
271 azimuths = -numpy.radians(self.data.yrange)+numpy.pi/2
273 azimuths = -numpy.radians(self.data.yrange)+numpy.pi/2
272 r, theta = numpy.meshgrid(zeniths, azimuths)
274 r, theta = numpy.meshgrid(zeniths, azimuths)
273 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
275 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
274 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
276 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
275 x = km2deg(x) + self.lon
277 x = km2deg(x) + self.lon
276 y = km2deg(y) + self.lat
278 y = km2deg(y) + self.lat
277 else:
279 else:
278 azimuths = numpy.radians(self.data.yrange)
280 azimuths = numpy.radians(self.data.yrange)
279 r, theta = numpy.meshgrid(zeniths, azimuths)
281 r, theta = numpy.meshgrid(zeniths, azimuths)
280 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
282 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
281 self.y = zeniths
283 self.y = zeniths
282
284
283 if ax.firsttime:
285 if ax.firsttime:
284 if self.zlimits is not None:
286 if self.zlimits is not None:
285 self.zmin, self.zmax = self.zlimits[n]
287 self.zmin, self.zmax = self.zlimits[n]
286 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
288 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
287 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
289 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
288 vmin=self.zmin,
290 vmin=self.zmin,
289 vmax=self.zmax,
291 vmax=self.zmax,
290 cmap=self.cmaps[n])
292 cmap=self.cmaps[n])
291 else:
293 else:
292 if self.zlimits is not None:
294 if self.zlimits is not None:
293 self.zmin, self.zmax = self.zlimits[n]
295 self.zmin, self.zmax = self.zlimits[n]
294 ax.collections.remove(ax.collections[0])
296 ax.collections.remove(ax.collections[0])
295 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
297 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
296 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
298 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
297 vmin=self.zmin,
299 vmin=self.zmin,
298 vmax=self.zmax,
300 vmax=self.zmax,
299 cmap=self.cmaps[n])
301 cmap=self.cmaps[n])
300
302
301 if self.mode == 'A':
303 if self.mode == 'A':
302 continue
304 continue
303
305
304 # plot district names
306 # plot district names
305 f = open('/data/workspace/schain_scripts/distrito.csv')
307 f = open('/data/workspace/schain_scripts/distrito.csv')
306 for line in f:
308 for line in f:
307 label, lon, lat = [s.strip() for s in line.split(',') if s]
309 label, lon, lat = [s.strip() for s in line.split(',') if s]
308 lat = float(lat)
310 lat = float(lat)
309 lon = float(lon)
311 lon = float(lon)
310 # ax.plot(lon, lat, '.b', ms=2)
312 # ax.plot(lon, lat, '.b', ms=2)
311 ax.text(lon, lat, label.decode('utf8'), ha='center',
313 ax.text(lon, lat, label.decode('utf8'), ha='center',
312 va='bottom', size='8', color='black')
314 va='bottom', size='8', color='black')
313
315
314 # plot limites
316 # plot limites
315 limites = []
317 limites = []
316 tmp = []
318 tmp = []
317 for line in open('/data/workspace/schain_scripts/lima.csv'):
319 for line in open('/data/workspace/schain_scripts/lima.csv'):
318 if '#' in line:
320 if '#' in line:
319 if tmp:
321 if tmp:
320 limites.append(tmp)
322 limites.append(tmp)
321 tmp = []
323 tmp = []
322 continue
324 continue
323 values = line.strip().split(',')
325 values = line.strip().split(',')
324 tmp.append((float(values[0]), float(values[1])))
326 tmp.append((float(values[0]), float(values[1])))
325 for points in limites:
327 for points in limites:
326 ax.add_patch(
328 ax.add_patch(
327 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
329 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
328
330
329 # plot Cuencas
331 # plot Cuencas
330 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
332 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
331 f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
333 f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
332 values = [line.strip().split(',') for line in f]
334 values = [line.strip().split(',') for line in f]
333 points = [(float(s[0]), float(s[1])) for s in values]
335 points = [(float(s[0]), float(s[1])) for s in values]
334 ax.add_patch(Polygon(points, ec='b', fc='none'))
336 ax.add_patch(Polygon(points, ec='b', fc='none'))
335
337
336 # plot grid
338 # plot grid
337 for r in (15, 30, 45, 60):
339 for r in (15, 30, 45, 60):
338 ax.add_artist(plt.Circle((self.lon, self.lat),
340 ax.add_artist(plt.Circle((self.lon, self.lat),
339 km2deg(r), color='0.6', fill=False, lw=0.2))
341 km2deg(r), color='0.6', fill=False, lw=0.2))
340 ax.text(
342 ax.text(
341 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
343 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
342 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
344 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
343 '{}km'.format(r),
345 '{}km'.format(r),
344 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
346 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
345
347
346 if self.mode == 'E':
348 if self.mode == 'E':
347 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
349 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
348 label = 'E{:02d}'.format(int(self.data.meta['elevation']))
350 label = 'E{:02d}'.format(int(self.data.meta['elevation']))
349 else:
351 else:
350 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
352 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
351 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
353 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
352
354
353 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
355 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
354 self.titles = ['{} {}'.format(
356 self.titles = ['{} {}'.format(
355 self.data.parameters[x], title) for x in self.channels]
357 self.data.parameters[x], title) for x in self.channels]
Show file before | Show file after | Hide comments
@@ -1,737 +1,961
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """Classes to plot Spectra data
5 """Classes to plot Spectra data
6
6
7 """
7 """
8
8
9 import os
9 import os
10 import numpy
10 import numpy
11
11
12 from schainpy.model.graphics.jroplot_base import Plot, plt, log
12 from schainpy.model.graphics.jroplot_base import Plot, plt, log
13 from itertools import combinations
13 from itertools import combinations
14
14
15
15
16 class SpectraPlot(Plot):
16 class SpectraPlot(Plot):
17 '''
17 '''
18 Plot for Spectra data
18 Plot for Spectra data
19 '''
19 '''
20
20
21 CODE = 'spc'
21 CODE = 'spc'
22 colormap = 'jet'
22 colormap = 'jet'
23 plot_type = 'pcolor'
23 plot_type = 'pcolor'
24 buffering = False
24 buffering = False
25 channelList = []
25 channelList = []
26
26
27 def setup(self):
27 def setup(self):
28
28 self.nplots = len(self.data.channels)
29 self.nplots = len(self.data.channels)
29 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
30 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
30 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
31 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
31 self.height = 2.6 * self.nrows
32 self.height = 2.6 * self.nrows
32
33
33 self.cb_label = 'dB'
34 self.cb_label = 'dB'
34 if self.showprofile:
35 if self.showprofile:
35 self.width = 4 * self.ncols
36 self.width = 4 * self.ncols
36 else:
37 else:
37 self.width = 3.5 * self.ncols
38 self.width = 3.5 * self.ncols
38 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08})
39 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08})
39 self.ylabel = 'Range [km]'
40 self.ylabel = 'Range [km]'
41
42
40 def update_list(self,dataOut):
43 def update_list(self,dataOut):
41 if len(self.channelList) == 0:
44 if len(self.channelList) == 0:
42 self.channelList = dataOut.channelList
45 self.channelList = dataOut.channelList
43
46
44 def update(self, dataOut):
47 def update(self, dataOut):
48
45 self.update_list(dataOut)
49 self.update_list(dataOut)
46 data = {}
50 data = {}
47 meta = {}
51 meta = {}
48 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
52 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
49
50 data['spc'] = spc
53 data['spc'] = spc
51 data['rti'] = dataOut.getPower()
54 data['rti'] = dataOut.getPower()
52 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
55 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
53 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
56 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
54 if self.CODE == 'spc_moments':
57 if self.CODE == 'spc_moments':
55 data['moments'] = dataOut.moments
58 data['moments'] = dataOut.moments
56
59
57 return data, meta
60 return data, meta
58
61
59 def plot(self):
62 def plot(self):
60 if self.xaxis == "frequency":
63 if self.xaxis == "frequency":
61 x = self.data.xrange[0]
64 x = self.data.xrange[0]
62 self.xlabel = "Frequency (kHz)"
65 self.xlabel = "Frequency (kHz)"
63 elif self.xaxis == "time":
66 elif self.xaxis == "time":
64 x = self.data.xrange[1]
67 x = self.data.xrange[1]
65 self.xlabel = "Time (ms)"
68 self.xlabel = "Time (ms)"
66 else:
69 else:
67 x = self.data.xrange[2]
70 x = self.data.xrange[2]
68 self.xlabel = "Velocity (m/s)"
71 self.xlabel = "Velocity (m/s)"
69
72
70 if self.CODE == 'spc_moments':
73 if self.CODE == 'spc_moments':
71 x = self.data.xrange[2]
74 x = self.data.xrange[2]
72 self.xlabel = "Velocity (m/s)"
75 self.xlabel = "Velocity (m/s)"
73
76
74 self.titles = []
77 self.titles = []
75
76 y = self.data.yrange
78 y = self.data.yrange
77 self.y = y
79 self.y = y
78
80
79 data = self.data[-1]
81 data = self.data[-1]
80 z = data['spc']
82 z = data['spc']
81
83
82 for n, ax in enumerate(self.axes):
84 for n, ax in enumerate(self.axes):
83 noise = data['noise'][n]
85 noise = data['noise'][n]
84 if self.CODE == 'spc_moments':
86 if self.CODE == 'spc_moments':
85 mean = data['moments'][n, 1]
87 mean = data['moments'][n, 1]
86 if ax.firsttime:
88 if ax.firsttime:
87 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
89 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
88 self.xmin = self.xmin if self.xmin else -self.xmax
90 self.xmin = self.xmin if self.xmin else -self.xmax
89 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
91 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
90 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
92 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
91 ax.plt = ax.pcolormesh(x, y, z[n].T,
93 ax.plt = ax.pcolormesh(x, y, z[n].T,
92 vmin=self.zmin,
94 vmin=self.zmin,
93 vmax=self.zmax,
95 vmax=self.zmax,
94 cmap=plt.get_cmap(self.colormap)
96 cmap=plt.get_cmap(self.colormap)
95 )
97 )
96
98
97 if self.showprofile:
99 if self.showprofile:
98 ax.plt_profile = self.pf_axes[n].plot(
100 ax.plt_profile = self.pf_axes[n].plot(
99 data['rti'][n], y)[0]
101 data['rti'][n], y)[0]
100 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
102 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
101 color="k", linestyle="dashed", lw=1)[0]
103 color="k", linestyle="dashed", lw=1)[0]
102 if self.CODE == 'spc_moments':
104 if self.CODE == 'spc_moments':
103 ax.plt_mean = ax.plot(mean, y, color='k')[0]
105 ax.plt_mean = ax.plot(mean, y, color='k')[0]
104 else:
106 else:
105 ax.plt.set_array(z[n].T.ravel())
107 ax.plt.set_array(z[n].T.ravel())
106 if self.showprofile:
108 if self.showprofile:
107 ax.plt_profile.set_data(data['rti'][n], y)
109 ax.plt_profile.set_data(data['rti'][n], y)
108 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
110 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
109 if self.CODE == 'spc_moments':
111 if self.CODE == 'spc_moments':
110 ax.plt_mean.set_data(mean, y)
112 ax.plt_mean.set_data(mean, y)
111 self.titles.append('CH {}: {:3.2f}dB'.format(self.channelList[n], noise))
113 self.titles.append('CH {}: {:3.2f}dB'.format(self.channelList[n], noise))
112
114
113
115
114 class CrossSpectraPlot(Plot):
116 class CrossSpectraPlot(Plot):
115
117
116 CODE = 'cspc'
118 CODE = 'cspc'
117 colormap = 'jet'
119 colormap = 'jet'
118 plot_type = 'pcolor'
120 plot_type = 'pcolor'
119 zmin_coh = None
121 zmin_coh = None
120 zmax_coh = None
122 zmax_coh = None
121 zmin_phase = None
123 zmin_phase = None
122 zmax_phase = None
124 zmax_phase = None
123 realChannels = None
125 realChannels = None
124 crossPairs = None
126 crossPairs = None
125
127
126 def setup(self):
128 def setup(self):
127
129
128 self.ncols = 4
130 self.ncols = 4
129 self.nplots = len(self.data.pairs) * 2
131 self.nplots = len(self.data.pairs) * 2
130 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
132 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
131 self.width = 3.1 * self.ncols
133 self.width = 3.1 * self.ncols
132 self.height = 2.6 * self.nrows
134 self.height = 2.6 * self.nrows
133 self.ylabel = 'Range [km]'
135 self.ylabel = 'Range [km]'
134 self.showprofile = False
136 self.showprofile = False
135 self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
137 self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
136
138
137 def update(self, dataOut):
139 def update(self, dataOut):
138
140
139 data = {}
141 data = {}
140 meta = {}
142 meta = {}
141
143
142 spc = dataOut.data_spc
144 spc = dataOut.data_spc
143 cspc = dataOut.data_cspc
145 cspc = dataOut.data_cspc
144 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
146 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
145 rawPairs = list(combinations(list(range(dataOut.nChannels)), 2))
147 rawPairs = list(combinations(list(range(dataOut.nChannels)), 2))
146 meta['pairs'] = rawPairs
148 meta['pairs'] = rawPairs
147
149
148 if self.crossPairs == None:
150 if self.crossPairs == None:
149 self.crossPairs = dataOut.pairsList
151 self.crossPairs = dataOut.pairsList
150
152
151 tmp = []
153 tmp = []
152
154
153 for n, pair in enumerate(meta['pairs']):
155 for n, pair in enumerate(meta['pairs']):
154
156
155 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
157 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
156 coh = numpy.abs(out)
158 coh = numpy.abs(out)
157 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
159 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
158 tmp.append(coh)
160 tmp.append(coh)
159 tmp.append(phase)
161 tmp.append(phase)
160
162
161 data['cspc'] = numpy.array(tmp)
163 data['cspc'] = numpy.array(tmp)
162
164
163 return data, meta
165 return data, meta
164
166
165 def plot(self):
167 def plot(self):
166
168
167 if self.xaxis == "frequency":
169 if self.xaxis == "frequency":
168 x = self.data.xrange[0]
170 x = self.data.xrange[0]
169 self.xlabel = "Frequency (kHz)"
171 self.xlabel = "Frequency (kHz)"
170 elif self.xaxis == "time":
172 elif self.xaxis == "time":
171 x = self.data.xrange[1]
173 x = self.data.xrange[1]
172 self.xlabel = "Time (ms)"
174 self.xlabel = "Time (ms)"
173 else:
175 else:
174 x = self.data.xrange[2]
176 x = self.data.xrange[2]
175 self.xlabel = "Velocity (m/s)"
177 self.xlabel = "Velocity (m/s)"
176
178
177 self.titles = []
179 self.titles = []
178
180
179 y = self.data.yrange
181 y = self.data.yrange
180 self.y = y
182 self.y = y
181
183
182 data = self.data[-1]
184 data = self.data[-1]
183 cspc = data['cspc']
185 cspc = data['cspc']
184
186
185 for n in range(len(self.data.pairs)):
187 for n in range(len(self.data.pairs)):
186
188
187 pair = self.crossPairs[n]
189 pair = self.crossPairs[n]
188
190
189 coh = cspc[n*2]
191 coh = cspc[n*2]
190 phase = cspc[n*2+1]
192 phase = cspc[n*2+1]
191 ax = self.axes[2 * n]
193 ax = self.axes[2 * n]
192
194
193 if ax.firsttime:
195 if ax.firsttime:
194 ax.plt = ax.pcolormesh(x, y, coh.T,
196 ax.plt = ax.pcolormesh(x, y, coh.T,
195 vmin=0,
197 vmin=self.zmin_coh,
196 vmax=1,
198 vmax=self.zmax_coh,
197 cmap=plt.get_cmap(self.colormap_coh)
199 cmap=plt.get_cmap(self.colormap_coh)
198 )
200 )
199 else:
201 else:
200 ax.plt.set_array(coh.T.ravel())
202 ax.plt.set_array(coh.T.ravel())
201 self.titles.append(
203 self.titles.append(
202 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
204 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
203
205
204 ax = self.axes[2 * n + 1]
206 ax = self.axes[2 * n + 1]
205 if ax.firsttime:
207 if ax.firsttime:
206 ax.plt = ax.pcolormesh(x, y, phase.T,
208 ax.plt = ax.pcolormesh(x, y, phase.T,
207 vmin=-180,
209 vmin=-180,
208 vmax=180,
210 vmax=180,
209 cmap=plt.get_cmap(self.colormap_phase)
211 cmap=plt.get_cmap(self.colormap_phase)
210 )
212 )
211 else:
213 else:
212 ax.plt.set_array(phase.T.ravel())
214 ax.plt.set_array(phase.T.ravel())
215
213 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
216 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
214
217
215
218
216 class RTIPlot(Plot):
219 class RTIPlot(Plot):
217 '''
220 '''
218 Plot for RTI data
221 Plot for RTI data
219 '''
222 '''
220
223
221 CODE = 'rti'
224 CODE = 'rti'
222 colormap = 'jet'
225 colormap = 'jet'
223 plot_type = 'pcolorbuffer'
226 plot_type = 'pcolorbuffer'
224 titles = None
227 titles = None
225 channelList = []
228 channelList = []
226
229
227 def setup(self):
230 def setup(self):
228 self.xaxis = 'time'
231 self.xaxis = 'time'
229 self.ncols = 1
232 self.ncols = 1
230 #print("dataChannels ",self.data.channels)
233 #print("dataChannels ",self.data.channels)
231 self.nrows = len(self.data.channels)
234 self.nrows = len(self.data.channels)
232 self.nplots = len(self.data.channels)
235 self.nplots = len(self.data.channels)
233 self.ylabel = 'Range [km]'
236 self.ylabel = 'Range [km]'
234 self.xlabel = 'Time'
237 self.xlabel = 'Time'
235 self.cb_label = 'dB'
238 self.cb_label = 'dB'
236 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95})
239 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95})
237 self.titles = ['{} Channel {}'.format(
240 self.titles = ['{} Channel {}'.format(
238 self.CODE.upper(), x) for x in range(self.nplots)]
241 self.CODE.upper(), x) for x in range(self.nplots)]
239
242
240 def update_list(self,dataOut):
243 def update_list(self,dataOut):
241
244
242 self.channelList = dataOut.channelList
245 self.channelList = dataOut.channelList
243
246
244
247
245 def update(self, dataOut):
248 def update(self, dataOut):
246 if len(self.channelList) == 0:
249 if len(self.channelList) == 0:
247 self.update_list(dataOut)
250 self.update_list(dataOut)
248 data = {}
251 data = {}
249 meta = {}
252 meta = {}
250 data['rti'] = dataOut.getPower()
253 data['rti'] = dataOut.getPower()
251 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
254 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
252 return data, meta
255 return data, meta
253
256
254 def plot(self):
257 def plot(self):
255
258
256 self.x = self.data.times
259 self.x = self.data.times
257 self.y = self.data.yrange
260 self.y = self.data.yrange
258 self.z = self.data[self.CODE]
261 self.z = self.data[self.CODE]
259 self.z = numpy.array(self.z, dtype=float)
262 self.z = numpy.array(self.z, dtype=float)
260 self.z = numpy.ma.masked_invalid(self.z)
263 self.z = numpy.ma.masked_invalid(self.z)
264
261 try:
265 try:
262 if self.channelList != None:
266 if self.channelList != None:
263 self.titles = ['{} Channel {}'.format(
267 self.titles = ['{} Channel {}'.format(
264 self.CODE.upper(), x) for x in self.channelList]
268 self.CODE.upper(), x) for x in self.channelList]
265 except:
269 except:
266 if self.channelList.any() != None:
270 if self.channelList.any() != None:
267 self.titles = ['{} Channel {}'.format(
271 self.titles = ['{} Channel {}'.format(
268 self.CODE.upper(), x) for x in self.channelList]
272 self.CODE.upper(), x) for x in self.channelList]
269 if self.decimation is None:
273 if self.decimation is None:
270 x, y, z = self.fill_gaps(self.x, self.y, self.z)
274 x, y, z = self.fill_gaps(self.x, self.y, self.z)
271 else:
275 else:
272 x, y, z = self.fill_gaps(*self.decimate())
276 x, y, z = self.fill_gaps(*self.decimate())
273 dummy_var = self.axes #ExtraΓ±amente esto actualiza el valor axes
277 dummy_var = self.axes #ExtraΓ±amente esto actualiza el valor axes
274 for n, ax in enumerate(self.axes):
278 for n, ax in enumerate(self.axes):
275 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
279 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
276 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
280 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
277 data = self.data[-1]
281 data = self.data[-1]
278 if ax.firsttime:
282 if ax.firsttime:
279 ax.plt = ax.pcolormesh(x, y, z[n].T,
283 ax.plt = ax.pcolormesh(x, y, z[n].T,
280 vmin=self.zmin,
284 vmin=self.zmin,
281 vmax=self.zmax,
285 vmax=self.zmax,
282 cmap=plt.get_cmap(self.colormap)
286 cmap=plt.get_cmap(self.colormap)
283 )
287 )
284 if self.showprofile:
288 if self.showprofile:
285 ax.plot_profile = self.pf_axes[n].plot(
289 ax.plot_profile = self.pf_axes[n].plot(data[self.CODE][n], self.y)[0]
286 data['rti'][n], self.y)[0]
290
287 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
291 if "noise" in self.data:
292 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
288 color="k", linestyle="dashed", lw=1)[0]
293 color="k", linestyle="dashed", lw=1)[0]
289 else:
294 else:
290 ax.collections.remove(ax.collections[0])
295 ax.collections.remove(ax.collections[0])
291 ax.plt = ax.pcolormesh(x, y, z[n].T,
296 ax.plt = ax.pcolormesh(x, y, z[n].T,
292 vmin=self.zmin,
297 vmin=self.zmin,
293 vmax=self.zmax,
298 vmax=self.zmax,
294 cmap=plt.get_cmap(self.colormap)
299 cmap=plt.get_cmap(self.colormap)
295 )
300 )
296 if self.showprofile:
301 if self.showprofile:
297 ax.plot_profile.set_data(data['rti'][n], self.y)
302 ax.plot_profile.set_data(data[self.CODE][n], self.y)
298 ax.plot_noise.set_data(numpy.repeat(
303 if "noise" in self.data:
304 ax.plot_noise.set_data(numpy.repeat(
299 data['noise'][n], len(self.y)), self.y)
305 data['noise'][n], len(self.y)), self.y)
300
306
301
307
302 class CoherencePlot(RTIPlot):
308 class CoherencePlot(RTIPlot):
303 '''
309 '''
304 Plot for Coherence data
310 Plot for Coherence data
305 '''
311 '''
306
312
307 CODE = 'coh'
313 CODE = 'coh'
308
314
309 def setup(self):
315 def setup(self):
310 self.xaxis = 'time'
316 self.xaxis = 'time'
311 self.ncols = 1
317 self.ncols = 1
312 self.nrows = len(self.data.pairs)
318 self.nrows = len(self.data.pairs)
313 self.nplots = len(self.data.pairs)
319 self.nplots = len(self.data.pairs)
314 self.ylabel = 'Range [km]'
320 self.ylabel = 'Range [km]'
315 self.xlabel = 'Time'
321 self.xlabel = 'Time'
316 self.plots_adjust.update({'hspace':0.6, 'left': 0.1, 'bottom': 0.1,'right':0.95})
322 self.plots_adjust.update({'hspace':0.6, 'left': 0.1, 'bottom': 0.1,'right':0.95})
317 if self.CODE == 'coh':
323 if self.CODE == 'coh':
318 self.cb_label = ''
324 self.cb_label = ''
319 self.titles = [
325 self.titles = [
320 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
326 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
321 else:
327 else:
322 self.cb_label = 'Degrees'
328 self.cb_label = 'Degrees'
323 self.titles = [
329 self.titles = [
324 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
330 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
325
331
326 def update(self, dataOut):
332 def update(self, dataOut):
327 self.update_list(dataOut)
333 self.update_list(dataOut)
328 data = {}
334 data = {}
329 meta = {}
335 meta = {}
330 data['coh'] = dataOut.getCoherence()
336 data['coh'] = dataOut.getCoherence()
331 meta['pairs'] = dataOut.pairsList
337 meta['pairs'] = dataOut.pairsList
332
338
333
339
334 return data, meta
340 return data, meta
335
341
336 class PhasePlot(CoherencePlot):
342 class PhasePlot(CoherencePlot):
337 '''
343 '''
338 Plot for Phase map data
344 Plot for Phase map data
339 '''
345 '''
340
346
341 CODE = 'phase'
347 CODE = 'phase'
342 colormap = 'seismic'
348 colormap = 'seismic'
343
349
344 def update(self, dataOut):
350 def update(self, dataOut):
345
351
346 data = {}
352 data = {}
347 meta = {}
353 meta = {}
348 data['phase'] = dataOut.getCoherence(phase=True)
354 data['phase'] = dataOut.getCoherence(phase=True)
349 meta['pairs'] = dataOut.pairsList
355 meta['pairs'] = dataOut.pairsList
350
356
351 return data, meta
357 return data, meta
352
358
353 class NoisePlot(Plot):
359 class NoisePlot(Plot):
354 '''
360 '''
355 Plot for noise
361 Plot for noise
356 '''
362 '''
357
363
358 CODE = 'noise'
364 CODE = 'noise'
359 plot_type = 'scatterbuffer'
365 plot_type = 'scatterbuffer'
360
366
361 def setup(self):
367 def setup(self):
362 self.xaxis = 'time'
368 self.xaxis = 'time'
363 self.ncols = 1
369 self.ncols = 1
364 self.nrows = 1
370 self.nrows = 1
365 self.nplots = 1
371 self.nplots = 1
366 self.ylabel = 'Intensity [dB]'
372 self.ylabel = 'Intensity [dB]'
367 self.xlabel = 'Time'
373 self.xlabel = 'Time'
368 self.titles = ['Noise']
374 self.titles = ['Noise']
369 self.colorbar = False
375 self.colorbar = False
370 self.plots_adjust.update({'right': 0.85 })
376 self.plots_adjust.update({'right': 0.85 })
371
377
372 def update(self, dataOut):
378 def update(self, dataOut):
373
379
374 data = {}
380 data = {}
375 meta = {}
381 meta = {}
376 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor).reshape(dataOut.nChannels, 1)
382 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor).reshape(dataOut.nChannels, 1)
377 meta['yrange'] = numpy.array([])
383 meta['yrange'] = numpy.array([])
378
384
379 return data, meta
385 return data, meta
380
386
381 def plot(self):
387 def plot(self):
382
388
383 x = self.data.times
389 x = self.data.times
384 xmin = self.data.min_time
390 xmin = self.data.min_time
385 xmax = xmin + self.xrange * 60 * 60
391 xmax = xmin + self.xrange * 60 * 60
386 Y = self.data['noise']
392 Y = self.data['noise']
387
393
388 if self.axes[0].firsttime:
394 if self.axes[0].firsttime:
389 self.ymin = numpy.nanmin(Y) - 5
395 self.ymin = numpy.nanmin(Y) - 5
390 self.ymax = numpy.nanmax(Y) + 5
396 self.ymax = numpy.nanmax(Y) + 5
391 for ch in self.data.channels:
397 for ch in self.data.channels:
392 y = Y[ch]
398 y = Y[ch]
393 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
399 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
394 plt.legend(bbox_to_anchor=(1.18, 1.0))
400 plt.legend(bbox_to_anchor=(1.18, 1.0))
395 else:
401 else:
396 for ch in self.data.channels:
402 for ch in self.data.channels:
397 y = Y[ch]
403 y = Y[ch]
398 self.axes[0].lines[ch].set_data(x, y)
404 self.axes[0].lines[ch].set_data(x, y)
399
405
400
406
401 class PowerProfilePlot(Plot):
407 class PowerProfilePlot(Plot):
402
408
403 CODE = 'pow_profile'
409 CODE = 'pow_profile'
404 plot_type = 'scatter'
410 plot_type = 'scatter'
405
411
406 def setup(self):
412 def setup(self):
407
413
408 self.ncols = 1
414 self.ncols = 1
409 self.nrows = 1
415 self.nrows = 1
410 self.nplots = 1
416 self.nplots = 1
411 self.height = 4
417 self.height = 4
412 self.width = 3
418 self.width = 3
413 self.ylabel = 'Range [km]'
419 self.ylabel = 'Range [km]'
414 self.xlabel = 'Intensity [dB]'
420 self.xlabel = 'Intensity [dB]'
415 self.titles = ['Power Profile']
421 self.titles = ['Power Profile']
416 self.colorbar = False
422 self.colorbar = False
417
423
418 def update(self, dataOut):
424 def update(self, dataOut):
419
425
420 data = {}
426 data = {}
421 meta = {}
427 meta = {}
422 data[self.CODE] = dataOut.getPower()
428 data[self.CODE] = dataOut.getPower()
423
429
424 return data, meta
430 return data, meta
425
431
426 def plot(self):
432 def plot(self):
427
433
428 y = self.data.yrange
434 y = self.data.yrange
429 self.y = y
435 self.y = y
430
436
431 x = self.data[-1][self.CODE]
437 x = self.data[-1][self.CODE]
432
438
433 if self.xmin is None: self.xmin = numpy.nanmin(x)*0.9
439 if self.xmin is None: self.xmin = numpy.nanmin(x)*0.9
434 if self.xmax is None: self.xmax = numpy.nanmax(x)*1.1
440 if self.xmax is None: self.xmax = numpy.nanmax(x)*1.1
435
441
436 if self.axes[0].firsttime:
442 if self.axes[0].firsttime:
437 for ch in self.data.channels:
443 for ch in self.data.channels:
438 self.axes[0].plot(x[ch], y, lw=1, label='Ch{}'.format(ch))
444 self.axes[0].plot(x[ch], y, lw=1, label='Ch{}'.format(ch))
439 plt.legend()
445 plt.legend()
440 else:
446 else:
441 for ch in self.data.channels:
447 for ch in self.data.channels:
442 self.axes[0].lines[ch].set_data(x[ch], y)
448 self.axes[0].lines[ch].set_data(x[ch], y)
443
449
444
450
445 class SpectraCutPlot(Plot):
451 class SpectraCutPlot(Plot):
446
452
447 CODE = 'spc_cut'
453 CODE = 'spc_cut'
448 plot_type = 'scatter'
454 plot_type = 'scatter'
449 buffering = False
455 buffering = False
456 heights = []
457 channelList = []
458 maintitle = "Spectra Cuts"
450
459
451 def setup(self):
460 def setup(self):
452
461
453 self.nplots = len(self.data.channels)
462 self.nplots = len(self.data.channels)
454 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
463 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
455 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
464 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
456 self.width = 3.4 * self.ncols + 1.5
465 self.width = 3.6 * self.ncols + 1.5
457 self.height = 3 * self.nrows
466 self.height = 3 * self.nrows
458 self.ylabel = 'Power [dB]'
467 self.ylabel = 'Power [dB]'
459 self.colorbar = False
468 self.colorbar = False
460 self.plots_adjust.update({'left':0.1, 'hspace':0.3, 'right': 0.75, 'bottom':0.08})
469 self.plots_adjust.update({'left':0.1, 'hspace':0.3, 'right': 0.75, 'bottom':0.08})
470 if self.selectedHeight:
471 self.maintitle = "Spectra Cut for %d km " %(int(self.selectedHeight))
461
472
462 def update(self, dataOut):
473 def update(self, dataOut):
474 if len(self.channelList) == 0:
475 self.channelList = dataOut.channelList
463
476
477 self.heights = dataOut.heightList
478 if self.selectedHeight:
479 index_list = numpy.where(self.heights >= self.selectedHeight)
480 self.height_index = index_list[0]
481 self.height_index = self.height_index[0]
482 #print(self.height_index)
464 data = {}
483 data = {}
465 meta = {}
484 meta = {}
466 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
485 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
467 data['spc'] = spc
486 if self.selectedHeight:
487 data['spc'] = spc[:,:,self.height_index]
488 else:
489 data['spc'] = spc
468 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
490 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
469
491
470 return data, meta
492 return data, meta
471
493
472 def plot(self):
494 def plot(self):
473 if self.xaxis == "frequency":
495 if self.xaxis == "frequency":
474 x = self.data.xrange[0][1:]
496 x = self.data.xrange[0][1:]
475 self.xlabel = "Frequency (kHz)"
497 self.xlabel = "Frequency (kHz)"
476 elif self.xaxis == "time":
498 elif self.xaxis == "time":
477 x = self.data.xrange[1]
499 x = self.data.xrange[1]
478 self.xlabel = "Time (ms)"
500 self.xlabel = "Time (ms)"
479 else:
501 else:
480 x = self.data.xrange[2]
502 x = self.data.xrange[2]
481 self.xlabel = "Velocity (m/s)"
503 self.xlabel = "Velocity (m/s)"
482
504
483 self.titles = []
505 self.titles = []
484
506
485 y = self.data.yrange
507 y = self.data.yrange
486 z = self.data[-1]['spc']
508 z = self.data[-1]['spc']
487
509 #print(z.shape)
488 if self.height_index:
510 if self.height_index:
489 index = numpy.array(self.height_index)
511 index = numpy.array(self.height_index)
490 else:
512 else:
491 index = numpy.arange(0, len(y), int((len(y))/9))
513 index = numpy.arange(0, len(y), int((len(y))/9))
492
514
493 for n, ax in enumerate(self.axes):
515 for n, ax in enumerate(self.axes):
494 if ax.firsttime:
516 if ax.firsttime:
495 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
517 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
496 self.xmin = self.xmin if self.xmin else -self.xmax
518 self.xmin = self.xmin if self.xmin else -self.xmax
497 self.ymin = self.ymin if self.ymin else numpy.nanmin(z)
519 self.ymin = self.ymin if self.ymin else numpy.nanmin(z)
498 self.ymax = self.ymax if self.ymax else numpy.nanmax(z)
520 self.ymax = self.ymax if self.ymax else numpy.nanmax(z)
499 ax.plt = ax.plot(x, z[n, :, index].T)
521 if self.selectedHeight:
500 labels = ['Range = {:2.1f}km'.format(y[i]) for i in index]
522 ax.plt = ax.plot(x, z[n,:])
501 self.figures[0].legend(ax.plt, labels, loc='center right')
523
524 else:
525 ax.plt = ax.plot(x, z[n, :, index].T)
526 labels = ['Range = {:2.1f}km'.format(y[i]) for i in index]
527 self.figures[0].legend(ax.plt, labels, loc='center right')
502 else:
528 else:
503 for i, line in enumerate(ax.plt):
529 for i, line in enumerate(ax.plt):
504 line.set_data(x, z[n, :, index[i]])
530 if self.selectedHeight:
505 self.titles.append('CH {}'.format(n))
531 line.set_data(x, z[n, :])
506
532 else:
533 line.set_data(x, z[n, :, index[i]])
534 self.titles.append('CH {}'.format(self.channelList[n]))
535 plt.suptitle(self.maintitle)
507
536
508 class BeaconPhase(Plot):
537 class BeaconPhase(Plot):
509
538
510 __isConfig = None
539 __isConfig = None
511 __nsubplots = None
540 __nsubplots = None
512
541
513 PREFIX = 'beacon_phase'
542 PREFIX = 'beacon_phase'
514
543
515 def __init__(self):
544 def __init__(self):
516 Plot.__init__(self)
545 Plot.__init__(self)
517 self.timerange = 24*60*60
546 self.timerange = 24*60*60
518 self.isConfig = False
547 self.isConfig = False
519 self.__nsubplots = 1
548 self.__nsubplots = 1
520 self.counter_imagwr = 0
549 self.counter_imagwr = 0
521 self.WIDTH = 800
550 self.WIDTH = 800
522 self.HEIGHT = 400
551 self.HEIGHT = 400
523 self.WIDTHPROF = 120
552 self.WIDTHPROF = 120
524 self.HEIGHTPROF = 0
553 self.HEIGHTPROF = 0
525 self.xdata = None
554 self.xdata = None
526 self.ydata = None
555 self.ydata = None
527
556
528 self.PLOT_CODE = BEACON_CODE
557 self.PLOT_CODE = BEACON_CODE
529
558
530 self.FTP_WEI = None
559 self.FTP_WEI = None
531 self.EXP_CODE = None
560 self.EXP_CODE = None
532 self.SUB_EXP_CODE = None
561 self.SUB_EXP_CODE = None
533 self.PLOT_POS = None
562 self.PLOT_POS = None
534
563
535 self.filename_phase = None
564 self.filename_phase = None
536
565
537 self.figfile = None
566 self.figfile = None
538
567
539 self.xmin = None
568 self.xmin = None
540 self.xmax = None
569 self.xmax = None
541
570
542 def getSubplots(self):
571 def getSubplots(self):
543
572
544 ncol = 1
573 ncol = 1
545 nrow = 1
574 nrow = 1
546
575
547 return nrow, ncol
576 return nrow, ncol
548
577
549 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
578 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
550
579
551 self.__showprofile = showprofile
580 self.__showprofile = showprofile
552 self.nplots = nplots
581 self.nplots = nplots
553
582
554 ncolspan = 7
583 ncolspan = 7
555 colspan = 6
584 colspan = 6
556 self.__nsubplots = 2
585 self.__nsubplots = 2
557
586
558 self.createFigure(id = id,
587 self.createFigure(id = id,
559 wintitle = wintitle,
588 wintitle = wintitle,
560 widthplot = self.WIDTH+self.WIDTHPROF,
589 widthplot = self.WIDTH+self.WIDTHPROF,
561 heightplot = self.HEIGHT+self.HEIGHTPROF,
590 heightplot = self.HEIGHT+self.HEIGHTPROF,
562 show=show)
591 show=show)
563
592
564 nrow, ncol = self.getSubplots()
593 nrow, ncol = self.getSubplots()
565
594
566 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
595 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
567
596
568 def save_phase(self, filename_phase):
597 def save_phase(self, filename_phase):
569 f = open(filename_phase,'w+')
598 f = open(filename_phase,'w+')
570 f.write('\n\n')
599 f.write('\n\n')
571 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
600 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
572 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
601 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
573 f.close()
602 f.close()
574
603
575 def save_data(self, filename_phase, data, data_datetime):
604 def save_data(self, filename_phase, data, data_datetime):
576 f=open(filename_phase,'a')
605 f=open(filename_phase,'a')
577 timetuple_data = data_datetime.timetuple()
606 timetuple_data = data_datetime.timetuple()
578 day = str(timetuple_data.tm_mday)
607 day = str(timetuple_data.tm_mday)
579 month = str(timetuple_data.tm_mon)
608 month = str(timetuple_data.tm_mon)
580 year = str(timetuple_data.tm_year)
609 year = str(timetuple_data.tm_year)
581 hour = str(timetuple_data.tm_hour)
610 hour = str(timetuple_data.tm_hour)
582 minute = str(timetuple_data.tm_min)
611 minute = str(timetuple_data.tm_min)
583 second = str(timetuple_data.tm_sec)
612 second = str(timetuple_data.tm_sec)
584 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
613 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
585 f.close()
614 f.close()
586
615
587 def plot(self):
616 def plot(self):
588 log.warning('TODO: Not yet implemented...')
617 log.warning('TODO: Not yet implemented...')
589
618
590 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
619 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
591 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
620 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
592 timerange=None,
621 timerange=None,
593 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
622 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
594 server=None, folder=None, username=None, password=None,
623 server=None, folder=None, username=None, password=None,
595 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
624 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
596
625
597 if dataOut.flagNoData:
626 if dataOut.flagNoData:
598 return dataOut
627 return dataOut
599
628
600 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
629 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
601 return
630 return
602
631
603 if pairsList == None:
632 if pairsList == None:
604 pairsIndexList = dataOut.pairsIndexList[:10]
633 pairsIndexList = dataOut.pairsIndexList[:10]
605 else:
634 else:
606 pairsIndexList = []
635 pairsIndexList = []
607 for pair in pairsList:
636 for pair in pairsList:
608 if pair not in dataOut.pairsList:
637 if pair not in dataOut.pairsList:
609 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
638 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
610 pairsIndexList.append(dataOut.pairsList.index(pair))
639 pairsIndexList.append(dataOut.pairsList.index(pair))
611
640
612 if pairsIndexList == []:
641 if pairsIndexList == []:
613 return
642 return
614
643
615 # if len(pairsIndexList) > 4:
644 # if len(pairsIndexList) > 4:
616 # pairsIndexList = pairsIndexList[0:4]
645 # pairsIndexList = pairsIndexList[0:4]
617
646
618 hmin_index = None
647 hmin_index = None
619 hmax_index = None
648 hmax_index = None
620
649
621 if hmin != None and hmax != None:
650 if hmin != None and hmax != None:
622 indexes = numpy.arange(dataOut.nHeights)
651 indexes = numpy.arange(dataOut.nHeights)
623 hmin_list = indexes[dataOut.heightList >= hmin]
652 hmin_list = indexes[dataOut.heightList >= hmin]
624 hmax_list = indexes[dataOut.heightList <= hmax]
653 hmax_list = indexes[dataOut.heightList <= hmax]
625
654
626 if hmin_list.any():
655 if hmin_list.any():
627 hmin_index = hmin_list[0]
656 hmin_index = hmin_list[0]
628
657
629 if hmax_list.any():
658 if hmax_list.any():
630 hmax_index = hmax_list[-1]+1
659 hmax_index = hmax_list[-1]+1
631
660
632 x = dataOut.getTimeRange()
661 x = dataOut.getTimeRange()
633
662
634 thisDatetime = dataOut.datatime
663 thisDatetime = dataOut.datatime
635
664
636 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
665 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
637 xlabel = "Local Time"
666 xlabel = "Local Time"
638 ylabel = "Phase (degrees)"
667 ylabel = "Phase (degrees)"
639
668
640 update_figfile = False
669 update_figfile = False
641
670
642 nplots = len(pairsIndexList)
671 nplots = len(pairsIndexList)
643 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
672 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
644 phase_beacon = numpy.zeros(len(pairsIndexList))
673 phase_beacon = numpy.zeros(len(pairsIndexList))
645 for i in range(nplots):
674 for i in range(nplots):
646 pair = dataOut.pairsList[pairsIndexList[i]]
675 pair = dataOut.pairsList[pairsIndexList[i]]
647 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
676 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
648 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
677 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
649 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
678 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
650 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
679 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
651 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
680 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
652
681
653 if dataOut.beacon_heiIndexList:
682 if dataOut.beacon_heiIndexList:
654 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
683 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
655 else:
684 else:
656 phase_beacon[i] = numpy.average(phase)
685 phase_beacon[i] = numpy.average(phase)
657
686
658 if not self.isConfig:
687 if not self.isConfig:
659
688
660 nplots = len(pairsIndexList)
689 nplots = len(pairsIndexList)
661
690
662 self.setup(id=id,
691 self.setup(id=id,
663 nplots=nplots,
692 nplots=nplots,
664 wintitle=wintitle,
693 wintitle=wintitle,
665 showprofile=showprofile,
694 showprofile=showprofile,
666 show=show)
695 show=show)
667
696
668 if timerange != None:
697 if timerange != None:
669 self.timerange = timerange
698 self.timerange = timerange
670
699
671 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
700 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
672
701
673 if ymin == None: ymin = 0
702 if ymin == None: ymin = 0
674 if ymax == None: ymax = 360
703 if ymax == None: ymax = 360
675
704
676 self.FTP_WEI = ftp_wei
705 self.FTP_WEI = ftp_wei
677 self.EXP_CODE = exp_code
706 self.EXP_CODE = exp_code
678 self.SUB_EXP_CODE = sub_exp_code
707 self.SUB_EXP_CODE = sub_exp_code
679 self.PLOT_POS = plot_pos
708 self.PLOT_POS = plot_pos
680
709
681 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
710 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
682 self.isConfig = True
711 self.isConfig = True
683 self.figfile = figfile
712 self.figfile = figfile
684 self.xdata = numpy.array([])
713 self.xdata = numpy.array([])
685 self.ydata = numpy.array([])
714 self.ydata = numpy.array([])
686
715
687 update_figfile = True
716 update_figfile = True
688
717
689 #open file beacon phase
718 #open file beacon phase
690 path = '%s%03d' %(self.PREFIX, self.id)
719 path = '%s%03d' %(self.PREFIX, self.id)
691 beacon_file = os.path.join(path,'%s.txt'%self.name)
720 beacon_file = os.path.join(path,'%s.txt'%self.name)
692 self.filename_phase = os.path.join(figpath,beacon_file)
721 self.filename_phase = os.path.join(figpath,beacon_file)
693 #self.save_phase(self.filename_phase)
722 #self.save_phase(self.filename_phase)
694
723
695
724
696 #store data beacon phase
725 #store data beacon phase
697 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
726 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
698
727
699 self.setWinTitle(title)
728 self.setWinTitle(title)
700
729
701
730
702 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
731 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
703
732
704 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
733 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
705
734
706 axes = self.axesList[0]
735 axes = self.axesList[0]
707
736
708 self.xdata = numpy.hstack((self.xdata, x[0:1]))
737 self.xdata = numpy.hstack((self.xdata, x[0:1]))
709
738
710 if len(self.ydata)==0:
739 if len(self.ydata)==0:
711 self.ydata = phase_beacon.reshape(-1,1)
740 self.ydata = phase_beacon.reshape(-1,1)
712 else:
741 else:
713 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
742 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
714
743
715
744
716 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
745 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
717 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
746 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
718 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
747 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
719 XAxisAsTime=True, grid='both'
748 XAxisAsTime=True, grid='both'
720 )
749 )
721
750
722 self.draw()
751 self.draw()
723
752
724 if dataOut.ltctime >= self.xmax:
753 if dataOut.ltctime >= self.xmax:
725 self.counter_imagwr = wr_period
754 self.counter_imagwr = wr_period
726 self.isConfig = False
755 self.isConfig = False
727 update_figfile = True
756 update_figfile = True
728
757
729 self.save(figpath=figpath,
758 self.save(figpath=figpath,
730 figfile=figfile,
759 figfile=figfile,
731 save=save,
760 save=save,
732 ftp=ftp,
761 ftp=ftp,
733 wr_period=wr_period,
762 wr_period=wr_period,
734 thisDatetime=thisDatetime,
763 thisDatetime=thisDatetime,
735 update_figfile=update_figfile)
764 update_figfile=update_figfile)
736
765
737 return dataOut
766 return dataOut
767
768 class NoiselessSpectraPlot(Plot):
769 '''
770 Plot for Spectra data, subtracting
771 the noise in all channels, using for
772 amisr-14 data
773 '''
774
775 CODE = 'noiseless_spc'
776 colormap = 'nipy_spectral'
777 plot_type = 'pcolor'
778 buffering = False
779 channelList = []
780
781 def setup(self):
782
783 self.nplots = len(self.data.channels)
784 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
785 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
786 self.height = 2.6 * self.nrows
787
788 self.cb_label = 'dB'
789 if self.showprofile:
790 self.width = 4 * self.ncols
791 else:
792 self.width = 3.5 * self.ncols
793 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08})
794 self.ylabel = 'Range [km]'
795
796
797 def update_list(self,dataOut):
798 if len(self.channelList) == 0:
799 self.channelList = dataOut.channelList
800
801 def update(self, dataOut):
802
803 self.update_list(dataOut)
804 data = {}
805 meta = {}
806 n0 = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
807 (nch, nff, nh) = dataOut.data_spc.shape
808 n1 = numpy.repeat(n0,nh, axis=0).reshape((nch,nh))
809 noise = numpy.repeat(n1,nff, axis=1).reshape((nch,nff,nh))
810 #print(noise.shape, "noise", noise)
811
812 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor) - noise
813
814 data['spc'] = spc
815 data['rti'] = dataOut.getPower() - n1
816
817 data['noise'] = n0
818 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
819
820 return data, meta
821
822 def plot(self):
823 if self.xaxis == "frequency":
824 x = self.data.xrange[0]
825 self.xlabel = "Frequency (kHz)"
826 elif self.xaxis == "time":
827 x = self.data.xrange[1]
828 self.xlabel = "Time (ms)"
829 else:
830 x = self.data.xrange[2]
831 self.xlabel = "Velocity (m/s)"
832
833 self.titles = []
834 y = self.data.yrange
835 self.y = y
836
837 data = self.data[-1]
838 z = data['spc']
839
840 for n, ax in enumerate(self.axes):
841 noise = data['noise'][n]
842
843 if ax.firsttime:
844 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
845 self.xmin = self.xmin if self.xmin else -self.xmax
846 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
847 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
848 ax.plt = ax.pcolormesh(x, y, z[n].T,
849 vmin=self.zmin,
850 vmax=self.zmax,
851 cmap=plt.get_cmap(self.colormap)
852 )
853
854 if self.showprofile:
855 ax.plt_profile = self.pf_axes[n].plot(
856 data['rti'][n], y)[0]
857 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
858 color="k", linestyle="dashed", lw=1)[0]
859
860 else:
861 ax.plt.set_array(z[n].T.ravel())
862 if self.showprofile:
863 ax.plt_profile.set_data(data['rti'][n], y)
864 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
865
866 self.titles.append('CH {}: {:3.2f}dB'.format(self.channelList[n], noise))
867
868
869 class NoiselessRTIPlot(Plot):
870 '''
871 Plot for RTI data
872 '''
873
874 CODE = 'noiseless_rti'
875 colormap = 'jet'
876 plot_type = 'pcolorbuffer'
877 titles = None
878 channelList = []
879
880 def setup(self):
881 self.xaxis = 'time'
882 self.ncols = 1
883 #print("dataChannels ",self.data.channels)
884 self.nrows = len(self.data.channels)
885 self.nplots = len(self.data.channels)
886 self.ylabel = 'Range [km]'
887 self.xlabel = 'Time'
888 self.cb_label = 'dB'
889 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95})
890 self.titles = ['{} Channel {}'.format(
891 self.CODE.upper(), x) for x in range(self.nplots)]
892
893 def update_list(self,dataOut):
894
895 self.channelList = dataOut.channelList
896
897
898 def update(self, dataOut):
899 if len(self.channelList) == 0:
900 self.update_list(dataOut)
901 data = {}
902 meta = {}
903
904 n0 = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
905 (nch, nff, nh) = dataOut.data_spc.shape
906 noise = numpy.repeat(n0,nh, axis=0).reshape((nch,nh))
907
908
909 data['noiseless_rti'] = dataOut.getPower() - noise
910 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
911 return data, meta
912
913 def plot(self):
914
915 self.x = self.data.times
916 self.y = self.data.yrange
917 self.z = self.data['noiseless_rti']
918 self.z = numpy.array(self.z, dtype=float)
919 self.z = numpy.ma.masked_invalid(self.z)
920
921 try:
922 if self.channelList != None:
923 self.titles = ['{} Channel {}'.format(
924 self.CODE.upper(), x) for x in self.channelList]
925 except:
926 if self.channelList.any() != None:
927 self.titles = ['{} Channel {}'.format(
928 self.CODE.upper(), x) for x in self.channelList]
929 if self.decimation is None:
930 x, y, z = self.fill_gaps(self.x, self.y, self.z)
931 else:
932 x, y, z = self.fill_gaps(*self.decimate())
933 dummy_var = self.axes #ExtraΓ±amente esto actualiza el valor axes
934 for n, ax in enumerate(self.axes):
935 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
936 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
937 data = self.data[-1]
938 if ax.firsttime:
939 ax.plt = ax.pcolormesh(x, y, z[n].T,
940 vmin=self.zmin,
941 vmax=self.zmax,
942 cmap=plt.get_cmap(self.colormap)
943 )
944 if self.showprofile:
945 ax.plot_profile = self.pf_axes[n].plot(data['noiseless_rti'][n], self.y)[0]
946
947 if "noise" in self.data:
948 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
949 color="k", linestyle="dashed", lw=1)[0]
950 else:
951 ax.collections.remove(ax.collections[0])
952 ax.plt = ax.pcolormesh(x, y, z[n].T,
953 vmin=self.zmin,
954 vmax=self.zmax,
955 cmap=plt.get_cmap(self.colormap)
956 )
957 if self.showprofile:
958 ax.plot_profile.set_data(data['noiseless_rti'][n], self.y)
959 if "noise" in self.data:
960 ax.plot_noise.set_data(numpy.repeat(
961 data['noise'][n], len(self.y)), self.y)
Show file before | Show file after | Hide comments
@@ -1,1577 +1,1580
1 """
1 """
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 """
5 """
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import inspect
12 import inspect
13 import time
13 import time
14 import datetime
14 import datetime
15 import zmq
15 import zmq
16
16
17 from schainpy.model.proc.jroproc_base import Operation, MPDecorator
17 from schainpy.model.proc.jroproc_base import Operation, MPDecorator
18 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
18 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
19 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
19 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
20 from schainpy.utils import log
20 from schainpy.utils import log
21 import schainpy.admin
21 import schainpy.admin
22
22
23 LOCALTIME = True
23 LOCALTIME = True
24 DT_DIRECTIVES = {
24 DT_DIRECTIVES = {
25 '%Y': 4,
25 '%Y': 4,
26 '%y': 2,
26 '%y': 2,
27 '%m': 2,
27 '%m': 2,
28 '%d': 2,
28 '%d': 2,
29 '%j': 3,
29 '%j': 3,
30 '%H': 2,
30 '%H': 2,
31 '%M': 2,
31 '%M': 2,
32 '%S': 2,
32 '%S': 2,
33 '%f': 6
33 '%f': 6
34 }
34 }
35
35
36
36
37 def isNumber(cad):
37 def isNumber(cad):
38 """
38 """
39 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
39 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
40
40
41 Excepciones:
41 Excepciones:
42 Si un determinado string no puede ser convertido a numero
42 Si un determinado string no puede ser convertido a numero
43 Input:
43 Input:
44 str, string al cual se le analiza para determinar si convertible a un numero o no
44 str, string al cual se le analiza para determinar si convertible a un numero o no
45
45
46 Return:
46 Return:
47 True : si el string es uno numerico
47 True : si el string es uno numerico
48 False : no es un string numerico
48 False : no es un string numerico
49 """
49 """
50 try:
50 try:
51 float(cad)
51 float(cad)
52 return True
52 return True
53 except:
53 except:
54 return False
54 return False
55
55
56
56
57 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
57 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
58 """
58 """
59 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
59 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
60
60
61 Inputs:
61 Inputs:
62 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
62 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
63
63
64 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
64 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
65 segundos contados desde 01/01/1970.
65 segundos contados desde 01/01/1970.
66 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
66 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
67 segundos contados desde 01/01/1970.
67 segundos contados desde 01/01/1970.
68
68
69 Return:
69 Return:
70 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
70 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
71 fecha especificado, de lo contrario retorna False.
71 fecha especificado, de lo contrario retorna False.
72
72
73 Excepciones:
73 Excepciones:
74 Si el archivo no existe o no puede ser abierto
74 Si el archivo no existe o no puede ser abierto
75 Si la cabecera no puede ser leida.
75 Si la cabecera no puede ser leida.
76
76
77 """
77 """
78 basicHeaderObj = BasicHeader(LOCALTIME)
78 basicHeaderObj = BasicHeader(LOCALTIME)
79
79
80 try:
80 try:
81 fp = open(filename, 'rb')
81 fp = open(filename, 'rb')
82 except IOError:
82 except IOError:
83 print("The file %s can't be opened" % (filename))
83 print("The file %s can't be opened" % (filename))
84 return 0
84 return 0
85
85
86 sts = basicHeaderObj.read(fp)
86 sts = basicHeaderObj.read(fp)
87 fp.close()
87 fp.close()
88
88
89 if not(sts):
89 if not(sts):
90 print("Skipping the file %s because it has not a valid header" % (filename))
90 print("Skipping the file %s because it has not a valid header" % (filename))
91 return 0
91 return 0
92
92
93 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
93 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
94 return 0
94 return 0
95
95
96 return 1
96 return 1
97
97
98
98
99 def isTimeInRange(thisTime, startTime, endTime):
99 def isTimeInRange(thisTime, startTime, endTime):
100 if endTime >= startTime:
100 if endTime >= startTime:
101 if (thisTime < startTime) or (thisTime > endTime):
101 if (thisTime < startTime) or (thisTime > endTime):
102 return 0
102 return 0
103 return 1
103 return 1
104 else:
104 else:
105 if (thisTime < startTime) and (thisTime > endTime):
105 if (thisTime < startTime) and (thisTime > endTime):
106 return 0
106 return 0
107 return 1
107 return 1
108
108
109
109
110 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
110 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
111 """
111 """
112 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
112 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
113
113
114 Inputs:
114 Inputs:
115 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
115 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
116
116
117 startDate : fecha inicial del rango seleccionado en formato datetime.date
117 startDate : fecha inicial del rango seleccionado en formato datetime.date
118
118
119 endDate : fecha final del rango seleccionado en formato datetime.date
119 endDate : fecha final del rango seleccionado en formato datetime.date
120
120
121 startTime : tiempo inicial del rango seleccionado en formato datetime.time
121 startTime : tiempo inicial del rango seleccionado en formato datetime.time
122
122
123 endTime : tiempo final del rango seleccionado en formato datetime.time
123 endTime : tiempo final del rango seleccionado en formato datetime.time
124
124
125 Return:
125 Return:
126 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
126 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
127 fecha especificado, de lo contrario retorna False.
127 fecha especificado, de lo contrario retorna False.
128
128
129 Excepciones:
129 Excepciones:
130 Si el archivo no existe o no puede ser abierto
130 Si el archivo no existe o no puede ser abierto
131 Si la cabecera no puede ser leida.
131 Si la cabecera no puede ser leida.
132
132
133 """
133 """
134
134
135 try:
135 try:
136 fp = open(filename, 'rb')
136 fp = open(filename, 'rb')
137 except IOError:
137 except IOError:
138 print("The file %s can't be opened" % (filename))
138 print("The file %s can't be opened" % (filename))
139 return None
139 return None
140
140
141 firstBasicHeaderObj = BasicHeader(LOCALTIME)
141 firstBasicHeaderObj = BasicHeader(LOCALTIME)
142 systemHeaderObj = SystemHeader()
142 systemHeaderObj = SystemHeader()
143 radarControllerHeaderObj = RadarControllerHeader()
143 radarControllerHeaderObj = RadarControllerHeader()
144 processingHeaderObj = ProcessingHeader()
144 processingHeaderObj = ProcessingHeader()
145
145
146 lastBasicHeaderObj = BasicHeader(LOCALTIME)
146 lastBasicHeaderObj = BasicHeader(LOCALTIME)
147
147
148 sts = firstBasicHeaderObj.read(fp)
148 sts = firstBasicHeaderObj.read(fp)
149
149
150 if not(sts):
150 if not(sts):
151 print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
151 print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
152 return None
152 return None
153
153
154 if not systemHeaderObj.read(fp):
154 if not systemHeaderObj.read(fp):
155 return None
155 return None
156
156
157 if not radarControllerHeaderObj.read(fp):
157 if not radarControllerHeaderObj.read(fp):
158 return None
158 return None
159
159
160 if not processingHeaderObj.read(fp):
160 if not processingHeaderObj.read(fp):
161 return None
161 return None
162
162
163 filesize = os.path.getsize(filename)
163 filesize = os.path.getsize(filename)
164
164
165 offset = processingHeaderObj.blockSize + 24 # header size
165 offset = processingHeaderObj.blockSize + 24 # header size
166
166
167 if filesize <= offset:
167 if filesize <= offset:
168 print("[Reading] %s: This file has not enough data" % filename)
168 print("[Reading] %s: This file has not enough data" % filename)
169 return None
169 return None
170
170
171 fp.seek(-offset, 2)
171 fp.seek(-offset, 2)
172
172
173 sts = lastBasicHeaderObj.read(fp)
173 sts = lastBasicHeaderObj.read(fp)
174
174
175 fp.close()
175 fp.close()
176
176
177 thisDatetime = lastBasicHeaderObj.datatime
177 thisDatetime = lastBasicHeaderObj.datatime
178 thisTime_last_block = thisDatetime.time()
178 thisTime_last_block = thisDatetime.time()
179
179
180 thisDatetime = firstBasicHeaderObj.datatime
180 thisDatetime = firstBasicHeaderObj.datatime
181 thisDate = thisDatetime.date()
181 thisDate = thisDatetime.date()
182 thisTime_first_block = thisDatetime.time()
182 thisTime_first_block = thisDatetime.time()
183
183
184 # General case
184 # General case
185 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
185 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
186 #-----------o----------------------------o-----------
186 #-----------o----------------------------o-----------
187 # startTime endTime
187 # startTime endTime
188
188
189 if endTime >= startTime:
189 if endTime >= startTime:
190 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
190 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
191 return None
191 return None
192
192
193 return thisDatetime
193 return thisDatetime
194
194
195 # If endTime < startTime then endTime belongs to the next day
195 # If endTime < startTime then endTime belongs to the next day
196
196
197 #<<<<<<<<<<<o o>>>>>>>>>>>
197 #<<<<<<<<<<<o o>>>>>>>>>>>
198 #-----------o----------------------------o-----------
198 #-----------o----------------------------o-----------
199 # endTime startTime
199 # endTime startTime
200
200
201 if (thisDate == startDate) and (thisTime_last_block < startTime):
201 if (thisDate == startDate) and (thisTime_last_block < startTime):
202 return None
202 return None
203
203
204 if (thisDate == endDate) and (thisTime_first_block > endTime):
204 if (thisDate == endDate) and (thisTime_first_block > endTime):
205 return None
205 return None
206
206
207 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
207 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
208 return None
208 return None
209
209
210 return thisDatetime
210 return thisDatetime
211
211
212
212
213 def isFolderInDateRange(folder, startDate=None, endDate=None):
213 def isFolderInDateRange(folder, startDate=None, endDate=None):
214 """
214 """
215 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
215 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
216
216
217 Inputs:
217 Inputs:
218 folder : nombre completo del directorio.
218 folder : nombre completo del directorio.
219 Su formato deberia ser "/path_root/?YYYYDDD"
219 Su formato deberia ser "/path_root/?YYYYDDD"
220
220
221 siendo:
221 siendo:
222 YYYY : Anio (ejemplo 2015)
222 YYYY : Anio (ejemplo 2015)
223 DDD : Dia del anio (ejemplo 305)
223 DDD : Dia del anio (ejemplo 305)
224
224
225 startDate : fecha inicial del rango seleccionado en formato datetime.date
225 startDate : fecha inicial del rango seleccionado en formato datetime.date
226
226
227 endDate : fecha final del rango seleccionado en formato datetime.date
227 endDate : fecha final del rango seleccionado en formato datetime.date
228
228
229 Return:
229 Return:
230 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
230 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
231 fecha especificado, de lo contrario retorna False.
231 fecha especificado, de lo contrario retorna False.
232 Excepciones:
232 Excepciones:
233 Si el directorio no tiene el formato adecuado
233 Si el directorio no tiene el formato adecuado
234 """
234 """
235
235
236 basename = os.path.basename(folder)
236 basename = os.path.basename(folder)
237
237
238 if not isRadarFolder(basename):
238 if not isRadarFolder(basename):
239 print("The folder %s has not the rigth format" % folder)
239 print("The folder %s has not the rigth format" % folder)
240 return 0
240 return 0
241
241
242 if startDate and endDate:
242 if startDate and endDate:
243 thisDate = getDateFromRadarFolder(basename)
243 thisDate = getDateFromRadarFolder(basename)
244
244
245 if thisDate < startDate:
245 if thisDate < startDate:
246 return 0
246 return 0
247
247
248 if thisDate > endDate:
248 if thisDate > endDate:
249 return 0
249 return 0
250
250
251 return 1
251 return 1
252
252
253
253
254 def isFileInDateRange(filename, startDate=None, endDate=None):
254 def isFileInDateRange(filename, startDate=None, endDate=None):
255 """
255 """
256 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
256 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
257
257
258 Inputs:
258 Inputs:
259 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
259 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
260
260
261 Su formato deberia ser "?YYYYDDDsss"
261 Su formato deberia ser "?YYYYDDDsss"
262
262
263 siendo:
263 siendo:
264 YYYY : Anio (ejemplo 2015)
264 YYYY : Anio (ejemplo 2015)
265 DDD : Dia del anio (ejemplo 305)
265 DDD : Dia del anio (ejemplo 305)
266 sss : set
266 sss : set
267
267
268 startDate : fecha inicial del rango seleccionado en formato datetime.date
268 startDate : fecha inicial del rango seleccionado en formato datetime.date
269
269
270 endDate : fecha final del rango seleccionado en formato datetime.date
270 endDate : fecha final del rango seleccionado en formato datetime.date
271
271
272 Return:
272 Return:
273 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
273 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
274 fecha especificado, de lo contrario retorna False.
274 fecha especificado, de lo contrario retorna False.
275 Excepciones:
275 Excepciones:
276 Si el archivo no tiene el formato adecuado
276 Si el archivo no tiene el formato adecuado
277 """
277 """
278
278
279 basename = os.path.basename(filename)
279 basename = os.path.basename(filename)
280
280
281 if not isRadarFile(basename):
281 if not isRadarFile(basename):
282 print("The filename %s has not the rigth format" % filename)
282 print("The filename %s has not the rigth format" % filename)
283 return 0
283 return 0
284
284
285 if startDate and endDate:
285 if startDate and endDate:
286 thisDate = getDateFromRadarFile(basename)
286 thisDate = getDateFromRadarFile(basename)
287
287
288 if thisDate < startDate:
288 if thisDate < startDate:
289 return 0
289 return 0
290
290
291 if thisDate > endDate:
291 if thisDate > endDate:
292 return 0
292 return 0
293
293
294 return 1
294 return 1
295
295
296
296
297 def getFileFromSet(path, ext, set):
297 def getFileFromSet(path, ext, set):
298 validFilelist = []
298 validFilelist = []
299 fileList = os.listdir(path)
299 fileList = os.listdir(path)
300
300
301 # 0 1234 567 89A BCDE
301 # 0 1234 567 89A BCDE
302 # H YYYY DDD SSS .ext
302 # H YYYY DDD SSS .ext
303
303
304 for thisFile in fileList:
304 for thisFile in fileList:
305 try:
305 try:
306 year = int(thisFile[1:5])
306 year = int(thisFile[1:5])
307 doy = int(thisFile[5:8])
307 doy = int(thisFile[5:8])
308 except:
308 except:
309 continue
309 continue
310
310
311 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
311 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
312 continue
312 continue
313
313
314 validFilelist.append(thisFile)
314 validFilelist.append(thisFile)
315
315
316 myfile = fnmatch.filter(
316 myfile = fnmatch.filter(
317 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
317 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
318
318
319 if len(myfile) != 0:
319 if len(myfile) != 0:
320 return myfile[0]
320 return myfile[0]
321 else:
321 else:
322 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
322 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
323 print('the filename %s does not exist' % filename)
323 print('the filename %s does not exist' % filename)
324 print('...going to the last file: ')
324 print('...going to the last file: ')
325
325
326 if validFilelist:
326 if validFilelist:
327 validFilelist = sorted(validFilelist, key=str.lower)
327 validFilelist = sorted(validFilelist, key=str.lower)
328 return validFilelist[-1]
328 return validFilelist[-1]
329
329
330 return None
330 return None
331
331
332
332
333 def getlastFileFromPath(path, ext):
333 def getlastFileFromPath(path, ext):
334 """
334 """
335 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
335 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
336 al final de la depuracion devuelve el ultimo file de la lista que quedo.
336 al final de la depuracion devuelve el ultimo file de la lista que quedo.
337
337
338 Input:
338 Input:
339 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
339 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
340 ext : extension de los files contenidos en una carpeta
340 ext : extension de los files contenidos en una carpeta
341
341
342 Return:
342 Return:
343 El ultimo file de una determinada carpeta, no se considera el path.
343 El ultimo file de una determinada carpeta, no se considera el path.
344 """
344 """
345 validFilelist = []
345 validFilelist = []
346 fileList = os.listdir(path)
346 fileList = os.listdir(path)
347
347
348 # 0 1234 567 89A BCDE
348 # 0 1234 567 89A BCDE
349 # H YYYY DDD SSS .ext
349 # H YYYY DDD SSS .ext
350
350
351 for thisFile in fileList:
351 for thisFile in fileList:
352
352
353 year = thisFile[1:5]
353 year = thisFile[1:5]
354 if not isNumber(year):
354 if not isNumber(year):
355 continue
355 continue
356
356
357 doy = thisFile[5:8]
357 doy = thisFile[5:8]
358 if not isNumber(doy):
358 if not isNumber(doy):
359 continue
359 continue
360
360
361 year = int(year)
361 year = int(year)
362 doy = int(doy)
362 doy = int(doy)
363
363
364 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
364 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
365 continue
365 continue
366
366
367 validFilelist.append(thisFile)
367 validFilelist.append(thisFile)
368
368
369 if validFilelist:
369 if validFilelist:
370 validFilelist = sorted(validFilelist, key=str.lower)
370 validFilelist = sorted(validFilelist, key=str.lower)
371 return validFilelist[-1]
371 return validFilelist[-1]
372
372
373 return None
373 return None
374
374
375
375
376 def isRadarFolder(folder):
376 def isRadarFolder(folder):
377 try:
377 try:
378 year = int(folder[1:5])
378 year = int(folder[1:5])
379 doy = int(folder[5:8])
379 doy = int(folder[5:8])
380 except:
380 except:
381 return 0
381 return 0
382
382
383 return 1
383 return 1
384
384
385
385
386 def isRadarFile(file):
386 def isRadarFile(file):
387 try:
387 try:
388 year = int(file[1:5])
388 year = int(file[1:5])
389 doy = int(file[5:8])
389 doy = int(file[5:8])
390 set = int(file[8:11])
390 set = int(file[8:11])
391 except:
391 except:
392 return 0
392 return 0
393
393
394 return 1
394 return 1
395
395
396
396
397 def getDateFromRadarFile(file):
397 def getDateFromRadarFile(file):
398 try:
398 try:
399 year = int(file[1:5])
399 year = int(file[1:5])
400 doy = int(file[5:8])
400 doy = int(file[5:8])
401 set = int(file[8:11])
401 set = int(file[8:11])
402 except:
402 except:
403 return None
403 return None
404
404
405 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
405 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
406 return thisDate
406 return thisDate
407
407
408
408
409 def getDateFromRadarFolder(folder):
409 def getDateFromRadarFolder(folder):
410 try:
410 try:
411 year = int(folder[1:5])
411 year = int(folder[1:5])
412 doy = int(folder[5:8])
412 doy = int(folder[5:8])
413 except:
413 except:
414 return None
414 return None
415
415
416 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
416 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
417 return thisDate
417 return thisDate
418
418
419 def parse_format(s, fmt):
419 def parse_format(s, fmt):
420
420
421 for i in range(fmt.count('%')):
421 for i in range(fmt.count('%')):
422 x = fmt.index('%')
422 x = fmt.index('%')
423 d = DT_DIRECTIVES[fmt[x:x+2]]
423 d = DT_DIRECTIVES[fmt[x:x+2]]
424 fmt = fmt.replace(fmt[x:x+2], s[x:x+d])
424 fmt = fmt.replace(fmt[x:x+2], s[x:x+d])
425 return fmt
425 return fmt
426
426
427 class Reader(object):
427 class Reader(object):
428
428
429 c = 3E8
429 c = 3E8
430 isConfig = False
430 isConfig = False
431 dtype = None
431 dtype = None
432 pathList = []
432 pathList = []
433 filenameList = []
433 filenameList = []
434 datetimeList = []
434 datetimeList = []
435 filename = None
435 filename = None
436 ext = None
436 ext = None
437 flagIsNewFile = 1
437 flagIsNewFile = 1
438 flagDiscontinuousBlock = 0
438 flagDiscontinuousBlock = 0
439 flagIsNewBlock = 0
439 flagIsNewBlock = 0
440 flagNoMoreFiles = 0
440 flagNoMoreFiles = 0
441 fp = None
441 fp = None
442 firstHeaderSize = 0
442 firstHeaderSize = 0
443 basicHeaderSize = 24
443 basicHeaderSize = 24
444 versionFile = 1103
444 versionFile = 1103
445 fileSize = None
445 fileSize = None
446 fileSizeByHeader = None
446 fileSizeByHeader = None
447 fileIndex = -1
447 fileIndex = -1
448 profileIndex = None
448 profileIndex = None
449 blockIndex = 0
449 blockIndex = 0
450 nTotalBlocks = 0
450 nTotalBlocks = 0
451 maxTimeStep = 30
451 maxTimeStep = 30
452 lastUTTime = None
452 lastUTTime = None
453 datablock = None
453 datablock = None
454 dataOut = None
454 dataOut = None
455 getByBlock = False
455 getByBlock = False
456 path = None
456 path = None
457 startDate = None
457 startDate = None
458 endDate = None
458 endDate = None
459 startTime = datetime.time(0, 0, 0)
459 startTime = datetime.time(0, 0, 0)
460 endTime = datetime.time(23, 59, 59)
460 endTime = datetime.time(23, 59, 59)
461 set = None
461 set = None
462 expLabel = ""
462 expLabel = ""
463 online = False
463 online = False
464 delay = 60
464 delay = 60
465 nTries = 3 # quantity tries
465 nTries = 3 # quantity tries
466 nFiles = 3 # number of files for searching
466 nFiles = 3 # number of files for searching
467 walk = True
467 walk = True
468 getblock = False
468 getblock = False
469 nTxs = 1
469 nTxs = 1
470 realtime = False
470 realtime = False
471 blocksize = 0
471 blocksize = 0
472 blocktime = None
472 blocktime = None
473 warnings = True
473 warnings = True
474 verbose = True
474 verbose = True
475 server = None
475 server = None
476 format = None
476 format = None
477 oneDDict = None
477 oneDDict = None
478 twoDDict = None
478 twoDDict = None
479 independentParam = None
479 independentParam = None
480 filefmt = None
480 filefmt = None
481 folderfmt = None
481 folderfmt = None
482 open_file = open
482 open_file = open
483 open_mode = 'rb'
483 open_mode = 'rb'
484
484
485 def run(self):
485 def run(self):
486
486
487 raise NotImplementedError
487 raise NotImplementedError
488
488
489 def getAllowedArgs(self):
489 def getAllowedArgs(self):
490 if hasattr(self, '__attrs__'):
490 if hasattr(self, '__attrs__'):
491 return self.__attrs__
491 return self.__attrs__
492 else:
492 else:
493 return inspect.getargspec(self.run).args
493 return inspect.getargspec(self.run).args
494
494
495 def set_kwargs(self, **kwargs):
495 def set_kwargs(self, **kwargs):
496
496
497 for key, value in kwargs.items():
497 for key, value in kwargs.items():
498 setattr(self, key, value)
498 setattr(self, key, value)
499
499
500 def find_folders(self, path, startDate, endDate, folderfmt, last=False):
500 def find_folders(self, path, startDate, endDate, folderfmt, last=False):
501
501
502 folders = [x for f in path.split(',')
502 folders = [x for f in path.split(',')
503 for x in os.listdir(f) if os.path.isdir(os.path.join(f, x))]
503 for x in os.listdir(f) if os.path.isdir(os.path.join(f, x))]
504 folders.sort()
504 folders.sort()
505
505
506 if last:
506 if last:
507 folders = [folders[-1]]
507 folders = [folders[-1]]
508
508
509 for folder in folders:
509 for folder in folders:
510 try:
510 try:
511 dt = datetime.datetime.strptime(parse_format(folder, folderfmt), folderfmt).date()
511 dt = datetime.datetime.strptime(parse_format(folder, folderfmt), folderfmt).date()
512 if dt >= startDate and dt <= endDate:
512 if dt >= startDate and dt <= endDate:
513 yield os.path.join(path, folder)
513 yield os.path.join(path, folder)
514 else:
514 else:
515 log.log('Skiping folder {}'.format(folder), self.name)
515 log.log('Skiping folder {}'.format(folder), self.name)
516 except Exception as e:
516 except Exception as e:
517 log.log('Skiping folder {}'.format(folder), self.name)
517 log.log('Skiping folder {}'.format(folder), self.name)
518 continue
518 continue
519 return
519 return
520
520
521 def find_files(self, folders, ext, filefmt, startDate=None, endDate=None,
521 def find_files(self, folders, ext, filefmt, startDate=None, endDate=None,
522 expLabel='', last=False):
522 expLabel='', last=False):
523
523
524 for path in folders:
524 for path in folders:
525 files = glob.glob1(path, '*{}'.format(ext))
525 files = glob.glob1(path, '*{}'.format(ext))
526 files.sort()
526 files.sort()
527 if last:
527 if last:
528 if files:
528 if files:
529 fo = files[-1]
529 fo = files[-1]
530 try:
530 try:
531 dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
531 dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
532 yield os.path.join(path, expLabel, fo)
532 yield os.path.join(path, expLabel, fo)
533 except Exception as e:
533 except Exception as e:
534 pass
534 pass
535 return
535 return
536 else:
536 else:
537 return
537 return
538
538
539 for fo in files:
539 for fo in files:
540 try:
540 try:
541 dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
541 dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
542 if dt >= startDate and dt <= endDate:
542 if dt >= startDate and dt <= endDate:
543 yield os.path.join(path, expLabel, fo)
543 yield os.path.join(path, expLabel, fo)
544 else:
544 else:
545 log.log('Skiping file {}'.format(fo), self.name)
545 log.log('Skiping file {}'.format(fo), self.name)
546 except Exception as e:
546 except Exception as e:
547 log.log('Skiping file {}'.format(fo), self.name)
547 log.log('Skiping file {}'.format(fo), self.name)
548 continue
548 continue
549
549
550 def searchFilesOffLine(self, path, startDate, endDate,
550 def searchFilesOffLine(self, path, startDate, endDate,
551 expLabel, ext, walk,
551 expLabel, ext, walk,
552 filefmt, folderfmt):
552 filefmt, folderfmt):
553 """Search files in offline mode for the given arguments
553 """Search files in offline mode for the given arguments
554
554
555 Return:
555 Return:
556 Generator of files
556 Generator of files
557 """
557 """
558
558
559 if walk:
559 if walk:
560 folders = self.find_folders(
560 folders = self.find_folders(
561 path, startDate, endDate, folderfmt)
561 path, startDate, endDate, folderfmt)
562 else:
562 else:
563 folders = path.split(',')
563 folders = path.split(',')
564
564
565 return self.find_files(
565 return self.find_files(
566 folders, ext, filefmt, startDate, endDate, expLabel)
566 folders, ext, filefmt, startDate, endDate, expLabel)
567
567
568 def searchFilesOnLine(self, path, startDate, endDate,
568 def searchFilesOnLine(self, path, startDate, endDate,
569 expLabel, ext, walk,
569 expLabel, ext, walk,
570 filefmt, folderfmt):
570 filefmt, folderfmt):
571 """Search for the last file of the last folder
571 """Search for the last file of the last folder
572
572
573 Arguments:
573 Arguments:
574 path : carpeta donde estan contenidos los files que contiene data
574 path : carpeta donde estan contenidos los files que contiene data
575 expLabel : Nombre del subexperimento (subfolder)
575 expLabel : Nombre del subexperimento (subfolder)
576 ext : extension de los files
576 ext : extension de los files
577 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
577 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
578
578
579 Return:
579 Return:
580 generator with the full path of last filename
580 generator with the full path of last filename
581 """
581 """
582
582
583 if walk:
583 if walk:
584 folders = self.find_folders(
584 folders = self.find_folders(
585 path, startDate, endDate, folderfmt, last=True)
585 path, startDate, endDate, folderfmt, last=True)
586 else:
586 else:
587 folders = path.split(',')
587 folders = path.split(',')
588
588
589 return self.find_files(
589 return self.find_files(
590 folders, ext, filefmt, startDate, endDate, expLabel, last=True)
590 folders, ext, filefmt, startDate, endDate, expLabel, last=True)
591
591
592 def setNextFile(self):
592 def setNextFile(self):
593 """Set the next file to be readed open it and parse de file header"""
593 """Set the next file to be readed open it and parse de file header"""
594
594
595 while True:
595 while True:
596 if self.fp != None:
596 if self.fp != None:
597 self.fp.close()
597 self.fp.close()
598
598
599 if self.online:
599 if self.online:
600 newFile = self.setNextFileOnline()
600 newFile = self.setNextFileOnline()
601 else:
601 else:
602 newFile = self.setNextFileOffline()
602 newFile = self.setNextFileOffline()
603
603
604 if not(newFile):
604 if not(newFile):
605 if self.online:
605 if self.online:
606 raise schainpy.admin.SchainError('Time to wait for new files reach')
606 raise schainpy.admin.SchainError('Time to wait for new files reach')
607 else:
607 else:
608 if self.fileIndex == -1:
608 if self.fileIndex == -1:
609 raise schainpy.admin.SchainWarning('No files found in the given path')
609 raise schainpy.admin.SchainWarning('No files found in the given path')
610 else:
610 else:
611 raise schainpy.admin.SchainWarning('No more files to read')
611 raise schainpy.admin.SchainWarning('No more files to read')
612
612
613 if self.verifyFile(self.filename):
613 if self.verifyFile(self.filename):
614 break
614 break
615
615
616 log.log('Opening file: %s' % self.filename, self.name)
616 log.log('Opening file: %s' % self.filename, self.name)
617
617
618 self.readFirstHeader()
618 self.readFirstHeader()
619 self.nReadBlocks = 0
619 self.nReadBlocks = 0
620
620
621 def setNextFileOnline(self):
621 def setNextFileOnline(self):
622 """Check for the next file to be readed in online mode.
622 """Check for the next file to be readed in online mode.
623
623
624 Set:
624 Set:
625 self.filename
625 self.filename
626 self.fp
626 self.fp
627 self.filesize
627 self.filesize
628
628
629 Return:
629 Return:
630 boolean
630 boolean
631
631
632 """
632 """
633 nextFile = True
633 nextFile = True
634 nextDay = False
634 nextDay = False
635
635
636 for nFiles in range(self.nFiles+1):
636 for nFiles in range(self.nFiles+1):
637 for nTries in range(self.nTries):
637 for nTries in range(self.nTries):
638 fullfilename, filename = self.checkForRealPath(nextFile, nextDay)
638 fullfilename, filename = self.checkForRealPath(nextFile, nextDay)
639 if fullfilename is not None:
639 if fullfilename is not None:
640 break
640 break
641 log.warning(
641 log.warning(
642 "Waiting %0.2f sec for the next file: \"%s\" , try %02d ..." % (self.delay, filename, nTries + 1),
642 "Waiting %0.2f sec for the next file: \"%s\" , try %02d ..." % (self.delay, filename, nTries + 1),
643 self.name)
643 self.name)
644 time.sleep(self.delay)
644 time.sleep(self.delay)
645 nextFile = False
645 nextFile = False
646 continue
646 continue
647
647
648 if fullfilename is not None:
648 if fullfilename is not None:
649 break
649 break
650
650
651 self.nTries = 1
651 self.nTries = 1
652 nextFile = True
652 nextFile = True
653
653
654 if nFiles == (self.nFiles - 1):
654 if nFiles == (self.nFiles - 1):
655 log.log('Trying with next day...', self.name)
655 log.log('Trying with next day...', self.name)
656 nextDay = True
656 nextDay = True
657 self.nTries = 3
657 self.nTries = 3
658
658
659 if fullfilename:
659 if fullfilename:
660 self.fileSize = os.path.getsize(fullfilename)
660 self.fileSize = os.path.getsize(fullfilename)
661 self.filename = fullfilename
661 self.filename = fullfilename
662 self.flagIsNewFile = 1
662 self.flagIsNewFile = 1
663 if self.fp != None:
663 if self.fp != None:
664 self.fp.close()
664 self.fp.close()
665 self.fp = self.open_file(fullfilename, self.open_mode)
665 self.fp = self.open_file(fullfilename, self.open_mode)
666 self.flagNoMoreFiles = 0
666 self.flagNoMoreFiles = 0
667 self.fileIndex += 1
667 self.fileIndex += 1
668 return 1
668 return 1
669 else:
669 else:
670 return 0
670 return 0
671
671
672 def setNextFileOffline(self):
672 def setNextFileOffline(self):
673 """Open the next file to be readed in offline mode"""
673 """Open the next file to be readed in offline mode"""
674
674
675 try:
675 try:
676 filename = next(self.filenameList)
676 filename = next(self.filenameList)
677 self.fileIndex +=1
677 self.fileIndex +=1
678 except StopIteration:
678 except StopIteration:
679 self.flagNoMoreFiles = 1
679 self.flagNoMoreFiles = 1
680 return 0
680 return 0
681
681
682 self.filename = filename
682 self.filename = filename
683 self.fileSize = os.path.getsize(filename)
683 self.fileSize = os.path.getsize(filename)
684 self.fp = self.open_file(filename, self.open_mode)
684 try:
685 self.fp = self.open_file(filename, self.open_mode)
686 except Exception as e:
687 raise schainpy.admin.SchainError("[Reading] Error in {} file, unable to open".format(filename))
685 self.flagIsNewFile = 1
688 self.flagIsNewFile = 1
686
689
687 return 1
690 return 1
688
691
689 @staticmethod
692 @staticmethod
690 def isDateTimeInRange(dt, startDate, endDate, startTime, endTime):
693 def isDateTimeInRange(dt, startDate, endDate, startTime, endTime):
691 """Check if the given datetime is in range"""
694 """Check if the given datetime is in range"""
692 startDateTime= datetime.datetime.combine(startDate,startTime)
695 startDateTime= datetime.datetime.combine(startDate,startTime)
693 endDateTime = datetime.datetime.combine(endDate,endTime)
696 endDateTime = datetime.datetime.combine(endDate,endTime)
694
697
695 if startDateTime <= dt <= endDateTime:
698 if startDateTime <= dt <= endDateTime:
696 return True
699 return True
697 return False
700 return False
698
701
699 def verifyFile(self, filename):
702 def verifyFile(self, filename):
700 """Check for a valid file
703 """Check for a valid file
701
704
702 Arguments:
705 Arguments:
703 filename -- full path filename
706 filename -- full path filename
704
707
705 Return:
708 Return:
706 boolean
709 boolean
707 """
710 """
708
711
709 return True
712 return True
710
713
711 def checkForRealPath(self, nextFile, nextDay):
714 def checkForRealPath(self, nextFile, nextDay):
712 """Check if the next file to be readed exists"""
715 """Check if the next file to be readed exists"""
713
716
714 raise NotImplementedError
717 raise NotImplementedError
715
718
716 def readFirstHeader(self):
719 def readFirstHeader(self):
717 """Parse the file header"""
720 """Parse the file header"""
718
721
719 pass
722 pass
720
723
721 def waitDataBlock(self, pointer_location, blocksize=None):
724 def waitDataBlock(self, pointer_location, blocksize=None):
722 """
725 """
723 """
726 """
724
727
725 currentPointer = pointer_location
728 currentPointer = pointer_location
726 if blocksize is None:
729 if blocksize is None:
727 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
730 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
728 else:
731 else:
729 neededSize = blocksize
732 neededSize = blocksize
730
733
731 for nTries in range(self.nTries):
734 for nTries in range(self.nTries):
732 self.fp.close()
735 self.fp.close()
733 self.fp = open(self.filename, 'rb')
736 self.fp = open(self.filename, 'rb')
734 self.fp.seek(currentPointer)
737 self.fp.seek(currentPointer)
735
738
736 self.fileSize = os.path.getsize(self.filename)
739 self.fileSize = os.path.getsize(self.filename)
737 currentSize = self.fileSize - currentPointer
740 currentSize = self.fileSize - currentPointer
738
741
739 if (currentSize >= neededSize):
742 if (currentSize >= neededSize):
740 return 1
743 return 1
741
744
742 log.warning(
745 log.warning(
743 "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
746 "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
744 self.name
747 self.name
745 )
748 )
746 time.sleep(self.delay)
749 time.sleep(self.delay)
747
750
748 return 0
751 return 0
749
752
750 class JRODataReader(Reader):
753 class JRODataReader(Reader):
751
754
752 utc = 0
755 utc = 0
753 nReadBlocks = 0
756 nReadBlocks = 0
754 foldercounter = 0
757 foldercounter = 0
755 firstHeaderSize = 0
758 firstHeaderSize = 0
756 basicHeaderSize = 24
759 basicHeaderSize = 24
757 __isFirstTimeOnline = 1
760 __isFirstTimeOnline = 1
758 filefmt = "*%Y%j***"
761 filefmt = "*%Y%j***"
759 folderfmt = "*%Y%j"
762 folderfmt = "*%Y%j"
760 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'online', 'delay', 'walk']
763 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'online', 'delay', 'walk']
761
764
762 def getDtypeWidth(self):
765 def getDtypeWidth(self):
763
766
764 dtype_index = get_dtype_index(self.dtype)
767 dtype_index = get_dtype_index(self.dtype)
765 dtype_width = get_dtype_width(dtype_index)
768 dtype_width = get_dtype_width(dtype_index)
766
769
767 return dtype_width
770 return dtype_width
768
771
769 def checkForRealPath(self, nextFile, nextDay):
772 def checkForRealPath(self, nextFile, nextDay):
770 """Check if the next file to be readed exists.
773 """Check if the next file to be readed exists.
771
774
772 Example :
775 Example :
773 nombre correcto del file es .../.../D2009307/P2009307367.ext
776 nombre correcto del file es .../.../D2009307/P2009307367.ext
774
777
775 Entonces la funcion prueba con las siguientes combinaciones
778 Entonces la funcion prueba con las siguientes combinaciones
776 .../.../y2009307367.ext
779 .../.../y2009307367.ext
777 .../.../Y2009307367.ext
780 .../.../Y2009307367.ext
778 .../.../x2009307/y2009307367.ext
781 .../.../x2009307/y2009307367.ext
779 .../.../x2009307/Y2009307367.ext
782 .../.../x2009307/Y2009307367.ext
780 .../.../X2009307/y2009307367.ext
783 .../.../X2009307/y2009307367.ext
781 .../.../X2009307/Y2009307367.ext
784 .../.../X2009307/Y2009307367.ext
782 siendo para este caso, la ultima combinacion de letras, identica al file buscado
785 siendo para este caso, la ultima combinacion de letras, identica al file buscado
783
786
784 Return:
787 Return:
785 str -- fullpath of the file
788 str -- fullpath of the file
786 """
789 """
787
790
788
791
789 if nextFile:
792 if nextFile:
790 self.set += 1
793 self.set += 1
791 if nextDay:
794 if nextDay:
792 self.set = 0
795 self.set = 0
793 self.doy += 1
796 self.doy += 1
794 foldercounter = 0
797 foldercounter = 0
795 prefixDirList = [None, 'd', 'D']
798 prefixDirList = [None, 'd', 'D']
796 if self.ext.lower() == ".r": # voltage
799 if self.ext.lower() == ".r": # voltage
797 prefixFileList = ['d', 'D']
800 prefixFileList = ['d', 'D']
798 elif self.ext.lower() == ".pdata": # spectra
801 elif self.ext.lower() == ".pdata": # spectra
799 prefixFileList = ['p', 'P']
802 prefixFileList = ['p', 'P']
800
803
801 # barrido por las combinaciones posibles
804 # barrido por las combinaciones posibles
802 for prefixDir in prefixDirList:
805 for prefixDir in prefixDirList:
803 thispath = self.path
806 thispath = self.path
804 if prefixDir != None:
807 if prefixDir != None:
805 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
808 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
806 if foldercounter == 0:
809 if foldercounter == 0:
807 thispath = os.path.join(self.path, "%s%04d%03d" %
810 thispath = os.path.join(self.path, "%s%04d%03d" %
808 (prefixDir, self.year, self.doy))
811 (prefixDir, self.year, self.doy))
809 else:
812 else:
810 thispath = os.path.join(self.path, "%s%04d%03d_%02d" % (
813 thispath = os.path.join(self.path, "%s%04d%03d_%02d" % (
811 prefixDir, self.year, self.doy, foldercounter))
814 prefixDir, self.year, self.doy, foldercounter))
812 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
815 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
813 # formo el nombre del file xYYYYDDDSSS.ext
816 # formo el nombre del file xYYYYDDDSSS.ext
814 filename = "%s%04d%03d%03d%s" % (prefixFile, self.year, self.doy, self.set, self.ext)
817 filename = "%s%04d%03d%03d%s" % (prefixFile, self.year, self.doy, self.set, self.ext)
815 fullfilename = os.path.join(
818 fullfilename = os.path.join(
816 thispath, filename)
819 thispath, filename)
817
820
818 if os.path.exists(fullfilename):
821 if os.path.exists(fullfilename):
819 return fullfilename, filename
822 return fullfilename, filename
820
823
821 return None, filename
824 return None, filename
822
825
823 def __waitNewBlock(self):
826 def __waitNewBlock(self):
824 """
827 """
825 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
828 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
826
829
827 Si el modo de lectura es OffLine siempre retorn 0
830 Si el modo de lectura es OffLine siempre retorn 0
828 """
831 """
829 if not self.online:
832 if not self.online:
830 return 0
833 return 0
831
834
832 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
835 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
833 return 0
836 return 0
834
837
835 currentPointer = self.fp.tell()
838 currentPointer = self.fp.tell()
836
839
837 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
840 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
838
841
839 for nTries in range(self.nTries):
842 for nTries in range(self.nTries):
840
843
841 self.fp.close()
844 self.fp.close()
842 self.fp = open(self.filename, 'rb')
845 self.fp = open(self.filename, 'rb')
843 self.fp.seek(currentPointer)
846 self.fp.seek(currentPointer)
844
847
845 self.fileSize = os.path.getsize(self.filename)
848 self.fileSize = os.path.getsize(self.filename)
846 currentSize = self.fileSize - currentPointer
849 currentSize = self.fileSize - currentPointer
847
850
848 if (currentSize >= neededSize):
851 if (currentSize >= neededSize):
849 self.basicHeaderObj.read(self.fp)
852 self.basicHeaderObj.read(self.fp)
850 return 1
853 return 1
851
854
852 if self.fileSize == self.fileSizeByHeader:
855 if self.fileSize == self.fileSizeByHeader:
853 # self.flagEoF = True
856 # self.flagEoF = True
854 return 0
857 return 0
855
858
856 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
859 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
857 time.sleep(self.delay)
860 time.sleep(self.delay)
858
861
859 return 0
862 return 0
860
863
861 def __setNewBlock(self):
864 def __setNewBlock(self):
862
865
863 if self.fp == None:
866 if self.fp == None:
864 return 0
867 return 0
865
868
866 if self.flagIsNewFile:
869 if self.flagIsNewFile:
867 self.lastUTTime = self.basicHeaderObj.utc
870 self.lastUTTime = self.basicHeaderObj.utc
868 return 1
871 return 1
869
872
870 if self.realtime:
873 if self.realtime:
871 self.flagDiscontinuousBlock = 1
874 self.flagDiscontinuousBlock = 1
872 if not(self.setNextFile()):
875 if not(self.setNextFile()):
873 return 0
876 return 0
874 else:
877 else:
875 return 1
878 return 1
876
879
877 currentSize = self.fileSize - self.fp.tell()
880 currentSize = self.fileSize - self.fp.tell()
878 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
881 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
879
882
880 if (currentSize >= neededSize):
883 if (currentSize >= neededSize):
881 self.basicHeaderObj.read(self.fp)
884 self.basicHeaderObj.read(self.fp)
882 self.lastUTTime = self.basicHeaderObj.utc
885 self.lastUTTime = self.basicHeaderObj.utc
883 return 1
886 return 1
884
887
885 if self.__waitNewBlock():
888 if self.__waitNewBlock():
886 self.lastUTTime = self.basicHeaderObj.utc
889 self.lastUTTime = self.basicHeaderObj.utc
887 return 1
890 return 1
888
891
889 if not(self.setNextFile()):
892 if not(self.setNextFile()):
890 return 0
893 return 0
891
894
892 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
895 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
893 self.lastUTTime = self.basicHeaderObj.utc
896 self.lastUTTime = self.basicHeaderObj.utc
894
897
895 self.flagDiscontinuousBlock = 0
898 self.flagDiscontinuousBlock = 0
896
899
897 if deltaTime > self.maxTimeStep:
900 if deltaTime > self.maxTimeStep:
898 self.flagDiscontinuousBlock = 1
901 self.flagDiscontinuousBlock = 1
899
902
900 return 1
903 return 1
901
904
902 def readNextBlock(self):
905 def readNextBlock(self):
903
906
904 while True:
907 while True:
905 if not(self.__setNewBlock()):
908 if not(self.__setNewBlock()):
906 continue
909 continue
907
910
908 if not(self.readBlock()):
911 if not(self.readBlock()):
909 return 0
912 return 0
910
913
911 self.getBasicHeader()
914 self.getBasicHeader()
912
915
913 if not self.isDateTimeInRange(self.dataOut.datatime, self.startDate, self.endDate, self.startTime, self.endTime):
916 if not self.isDateTimeInRange(self.dataOut.datatime, self.startDate, self.endDate, self.startTime, self.endTime):
914 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
917 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
915 self.processingHeaderObj.dataBlocksPerFile,
918 self.processingHeaderObj.dataBlocksPerFile,
916 self.dataOut.datatime.ctime()))
919 self.dataOut.datatime.ctime()))
917 continue
920 continue
918
921
919 break
922 break
920
923
921 if self.verbose:
924 if self.verbose:
922 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
925 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
923 self.processingHeaderObj.dataBlocksPerFile,
926 self.processingHeaderObj.dataBlocksPerFile,
924 self.dataOut.datatime.ctime()))
927 self.dataOut.datatime.ctime()))
925 return 1
928 return 1
926
929
927 def readFirstHeader(self):
930 def readFirstHeader(self):
928
931
929 self.basicHeaderObj.read(self.fp)
932 self.basicHeaderObj.read(self.fp)
930 self.systemHeaderObj.read(self.fp)
933 self.systemHeaderObj.read(self.fp)
931 self.radarControllerHeaderObj.read(self.fp)
934 self.radarControllerHeaderObj.read(self.fp)
932 self.processingHeaderObj.read(self.fp)
935 self.processingHeaderObj.read(self.fp)
933 self.firstHeaderSize = self.basicHeaderObj.size
936 self.firstHeaderSize = self.basicHeaderObj.size
934
937
935 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
938 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
936 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
939 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
937 if datatype == 0:
940 if datatype == 0:
938 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
941 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
939 elif datatype == 1:
942 elif datatype == 1:
940 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
943 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
941 elif datatype == 2:
944 elif datatype == 2:
942 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
945 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
943 elif datatype == 3:
946 elif datatype == 3:
944 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
947 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
945 elif datatype == 4:
948 elif datatype == 4:
946 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
949 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
947 elif datatype == 5:
950 elif datatype == 5:
948 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
951 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
949 else:
952 else:
950 raise ValueError('Data type was not defined')
953 raise ValueError('Data type was not defined')
951
954
952 self.dtype = datatype_str
955 self.dtype = datatype_str
953 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
956 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
954 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
957 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
955 self.firstHeaderSize + self.basicHeaderSize * \
958 self.firstHeaderSize + self.basicHeaderSize * \
956 (self.processingHeaderObj.dataBlocksPerFile - 1)
959 (self.processingHeaderObj.dataBlocksPerFile - 1)
957 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
960 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
958 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
961 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
959 self.getBlockDimension()
962 self.getBlockDimension()
960
963
961 def verifyFile(self, filename):
964 def verifyFile(self, filename):
962
965
963 flag = True
966 flag = True
964
967
965 try:
968 try:
966 fp = open(filename, 'rb')
969 fp = open(filename, 'rb')
967 except IOError:
970 except IOError:
968 log.error("File {} can't be opened".format(filename), self.name)
971 log.error("File {} can't be opened".format(filename), self.name)
969 return False
972 return False
970
973
971 if self.online and self.waitDataBlock(0):
974 if self.online and self.waitDataBlock(0):
972 pass
975 pass
973
976
974 basicHeaderObj = BasicHeader(LOCALTIME)
977 basicHeaderObj = BasicHeader(LOCALTIME)
975 systemHeaderObj = SystemHeader()
978 systemHeaderObj = SystemHeader()
976 radarControllerHeaderObj = RadarControllerHeader()
979 radarControllerHeaderObj = RadarControllerHeader()
977 processingHeaderObj = ProcessingHeader()
980 processingHeaderObj = ProcessingHeader()
978
981
979 if not(basicHeaderObj.read(fp)):
982 if not(basicHeaderObj.read(fp)):
980 flag = False
983 flag = False
981 if not(systemHeaderObj.read(fp)):
984 if not(systemHeaderObj.read(fp)):
982 flag = False
985 flag = False
983 if not(radarControllerHeaderObj.read(fp)):
986 if not(radarControllerHeaderObj.read(fp)):
984 flag = False
987 flag = False
985 if not(processingHeaderObj.read(fp)):
988 if not(processingHeaderObj.read(fp)):
986 flag = False
989 flag = False
987 if not self.online:
990 if not self.online:
988 dt1 = basicHeaderObj.datatime
991 dt1 = basicHeaderObj.datatime
989 pos = self.fileSize-processingHeaderObj.blockSize-24
992 pos = self.fileSize-processingHeaderObj.blockSize-24
990 if pos<0:
993 if pos<0:
991 flag = False
994 flag = False
992 log.error('Invalid size for file: {}'.format(self.filename), self.name)
995 log.error('Invalid size for file: {}'.format(self.filename), self.name)
993 else:
996 else:
994 fp.seek(pos)
997 fp.seek(pos)
995 if not(basicHeaderObj.read(fp)):
998 if not(basicHeaderObj.read(fp)):
996 flag = False
999 flag = False
997 dt2 = basicHeaderObj.datatime
1000 dt2 = basicHeaderObj.datatime
998 if not self.isDateTimeInRange(dt1, self.startDate, self.endDate, self.startTime, self.endTime) and not \
1001 if not self.isDateTimeInRange(dt1, self.startDate, self.endDate, self.startTime, self.endTime) and not \
999 self.isDateTimeInRange(dt2, self.startDate, self.endDate, self.startTime, self.endTime):
1002 self.isDateTimeInRange(dt2, self.startDate, self.endDate, self.startTime, self.endTime):
1000 flag = False
1003 flag = False
1001
1004
1002 fp.close()
1005 fp.close()
1003 return flag
1006 return flag
1004
1007
1005 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1008 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1006
1009
1007 path_empty = True
1010 path_empty = True
1008
1011
1009 dateList = []
1012 dateList = []
1010 pathList = []
1013 pathList = []
1011
1014
1012 multi_path = path.split(',')
1015 multi_path = path.split(',')
1013
1016
1014 if not walk:
1017 if not walk:
1015
1018
1016 for single_path in multi_path:
1019 for single_path in multi_path:
1017
1020
1018 if not os.path.isdir(single_path):
1021 if not os.path.isdir(single_path):
1019 continue
1022 continue
1020
1023
1021 fileList = glob.glob1(single_path, "*" + ext)
1024 fileList = glob.glob1(single_path, "*" + ext)
1022
1025
1023 if not fileList:
1026 if not fileList:
1024 continue
1027 continue
1025
1028
1026 path_empty = False
1029 path_empty = False
1027
1030
1028 fileList.sort()
1031 fileList.sort()
1029
1032
1030 for thisFile in fileList:
1033 for thisFile in fileList:
1031
1034
1032 if not os.path.isfile(os.path.join(single_path, thisFile)):
1035 if not os.path.isfile(os.path.join(single_path, thisFile)):
1033 continue
1036 continue
1034
1037
1035 if not isRadarFile(thisFile):
1038 if not isRadarFile(thisFile):
1036 continue
1039 continue
1037
1040
1038 if not isFileInDateRange(thisFile, startDate, endDate):
1041 if not isFileInDateRange(thisFile, startDate, endDate):
1039 continue
1042 continue
1040
1043
1041 thisDate = getDateFromRadarFile(thisFile)
1044 thisDate = getDateFromRadarFile(thisFile)
1042
1045
1043 if thisDate in dateList or single_path in pathList:
1046 if thisDate in dateList or single_path in pathList:
1044 continue
1047 continue
1045
1048
1046 dateList.append(thisDate)
1049 dateList.append(thisDate)
1047 pathList.append(single_path)
1050 pathList.append(single_path)
1048
1051
1049 else:
1052 else:
1050 for single_path in multi_path:
1053 for single_path in multi_path:
1051
1054
1052 if not os.path.isdir(single_path):
1055 if not os.path.isdir(single_path):
1053 continue
1056 continue
1054
1057
1055 dirList = []
1058 dirList = []
1056
1059
1057 for thisPath in os.listdir(single_path):
1060 for thisPath in os.listdir(single_path):
1058
1061
1059 if not os.path.isdir(os.path.join(single_path, thisPath)):
1062 if not os.path.isdir(os.path.join(single_path, thisPath)):
1060 continue
1063 continue
1061
1064
1062 if not isRadarFolder(thisPath):
1065 if not isRadarFolder(thisPath):
1063 continue
1066 continue
1064
1067
1065 if not isFolderInDateRange(thisPath, startDate, endDate):
1068 if not isFolderInDateRange(thisPath, startDate, endDate):
1066 continue
1069 continue
1067
1070
1068 dirList.append(thisPath)
1071 dirList.append(thisPath)
1069
1072
1070 if not dirList:
1073 if not dirList:
1071 continue
1074 continue
1072
1075
1073 dirList.sort()
1076 dirList.sort()
1074
1077
1075 for thisDir in dirList:
1078 for thisDir in dirList:
1076
1079
1077 datapath = os.path.join(single_path, thisDir, expLabel)
1080 datapath = os.path.join(single_path, thisDir, expLabel)
1078 fileList = glob.glob1(datapath, "*" + ext)
1081 fileList = glob.glob1(datapath, "*" + ext)
1079
1082
1080 if not fileList:
1083 if not fileList:
1081 continue
1084 continue
1082
1085
1083 path_empty = False
1086 path_empty = False
1084
1087
1085 thisDate = getDateFromRadarFolder(thisDir)
1088 thisDate = getDateFromRadarFolder(thisDir)
1086
1089
1087 pathList.append(datapath)
1090 pathList.append(datapath)
1088 dateList.append(thisDate)
1091 dateList.append(thisDate)
1089
1092
1090 dateList.sort()
1093 dateList.sort()
1091
1094
1092 if walk:
1095 if walk:
1093 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1096 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1094 else:
1097 else:
1095 pattern_path = multi_path[0]
1098 pattern_path = multi_path[0]
1096
1099
1097 if path_empty:
1100 if path_empty:
1098 raise schainpy.admin.SchainError("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1101 raise schainpy.admin.SchainError("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1099 else:
1102 else:
1100 if not dateList:
1103 if not dateList:
1101 raise schainpy.admin.SchainError("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1104 raise schainpy.admin.SchainError("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1102
1105
1103 if include_path:
1106 if include_path:
1104 return dateList, pathList
1107 return dateList, pathList
1105
1108
1106 return dateList
1109 return dateList
1107
1110
1108 def setup(self, **kwargs):
1111 def setup(self, **kwargs):
1109
1112
1110 self.set_kwargs(**kwargs)
1113 self.set_kwargs(**kwargs)
1111 if not self.ext.startswith('.'):
1114 if not self.ext.startswith('.'):
1112 self.ext = '.{}'.format(self.ext)
1115 self.ext = '.{}'.format(self.ext)
1113
1116
1114 if self.server is not None:
1117 if self.server is not None:
1115 if 'tcp://' in self.server:
1118 if 'tcp://' in self.server:
1116 address = server
1119 address = server
1117 else:
1120 else:
1118 address = 'ipc:///tmp/%s' % self.server
1121 address = 'ipc:///tmp/%s' % self.server
1119 self.server = address
1122 self.server = address
1120 self.context = zmq.Context()
1123 self.context = zmq.Context()
1121 self.receiver = self.context.socket(zmq.PULL)
1124 self.receiver = self.context.socket(zmq.PULL)
1122 self.receiver.connect(self.server)
1125 self.receiver.connect(self.server)
1123 time.sleep(0.5)
1126 time.sleep(0.5)
1124 print('[Starting] ReceiverData from {}'.format(self.server))
1127 print('[Starting] ReceiverData from {}'.format(self.server))
1125 else:
1128 else:
1126 self.server = None
1129 self.server = None
1127 if self.path == None:
1130 if self.path == None:
1128 raise ValueError("[Reading] The path is not valid")
1131 raise ValueError("[Reading] The path is not valid")
1129
1132
1130 if self.online:
1133 if self.online:
1131 log.log("[Reading] Searching files in online mode...", self.name)
1134 log.log("[Reading] Searching files in online mode...", self.name)
1132
1135
1133 for nTries in range(self.nTries):
1136 for nTries in range(self.nTries):
1134 fullpath = self.searchFilesOnLine(self.path, self.startDate,
1137 fullpath = self.searchFilesOnLine(self.path, self.startDate,
1135 self.endDate, self.expLabel, self.ext, self.walk,
1138 self.endDate, self.expLabel, self.ext, self.walk,
1136 self.filefmt, self.folderfmt)
1139 self.filefmt, self.folderfmt)
1137
1140
1138 try:
1141 try:
1139 fullpath = next(fullpath)
1142 fullpath = next(fullpath)
1140 except:
1143 except:
1141 fullpath = None
1144 fullpath = None
1142
1145
1143 if fullpath:
1146 if fullpath:
1144 break
1147 break
1145
1148
1146 log.warning(
1149 log.warning(
1147 'Waiting {} sec for a valid file in {}: try {} ...'.format(
1150 'Waiting {} sec for a valid file in {}: try {} ...'.format(
1148 self.delay, self.path, nTries + 1),
1151 self.delay, self.path, nTries + 1),
1149 self.name)
1152 self.name)
1150 time.sleep(self.delay)
1153 time.sleep(self.delay)
1151
1154
1152 if not(fullpath):
1155 if not(fullpath):
1153 raise schainpy.admin.SchainError(
1156 raise schainpy.admin.SchainError(
1154 'There isn\'t any valid file in {}'.format(self.path))
1157 'There isn\'t any valid file in {}'.format(self.path))
1155
1158
1156 pathname, filename = os.path.split(fullpath)
1159 pathname, filename = os.path.split(fullpath)
1157 self.year = int(filename[1:5])
1160 self.year = int(filename[1:5])
1158 self.doy = int(filename[5:8])
1161 self.doy = int(filename[5:8])
1159 self.set = int(filename[8:11]) - 1
1162 self.set = int(filename[8:11]) - 1
1160 else:
1163 else:
1161 log.log("Searching files in {}".format(self.path), self.name)
1164 log.log("Searching files in {}".format(self.path), self.name)
1162 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
1165 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
1163 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
1166 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
1164
1167
1165 self.setNextFile()
1168 self.setNextFile()
1166
1169
1167 return
1170 return
1168
1171
1169 def getBasicHeader(self):
1172 def getBasicHeader(self):
1170
1173
1171 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1174 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1172 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1175 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1173
1176
1174 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1177 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1175
1178
1176 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1179 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1177
1180
1178 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1181 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1179
1182
1180 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1183 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1181
1184
1182 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1185 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1183
1186
1184 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1187 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1185
1188
1186 def getFirstHeader(self):
1189 def getFirstHeader(self):
1187
1190
1188 raise NotImplementedError
1191 raise NotImplementedError
1189
1192
1190 def getData(self):
1193 def getData(self):
1191
1194
1192 raise NotImplementedError
1195 raise NotImplementedError
1193
1196
1194 def hasNotDataInBuffer(self):
1197 def hasNotDataInBuffer(self):
1195
1198
1196 raise NotImplementedError
1199 raise NotImplementedError
1197
1200
1198 def readBlock(self):
1201 def readBlock(self):
1199
1202
1200 raise NotImplementedError
1203 raise NotImplementedError
1201
1204
1202 def isEndProcess(self):
1205 def isEndProcess(self):
1203
1206
1204 return self.flagNoMoreFiles
1207 return self.flagNoMoreFiles
1205
1208
1206 def printReadBlocks(self):
1209 def printReadBlocks(self):
1207
1210
1208 print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
1211 print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
1209
1212
1210 def printTotalBlocks(self):
1213 def printTotalBlocks(self):
1211
1214
1212 print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
1215 print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
1213
1216
1214 def run(self, **kwargs):
1217 def run(self, **kwargs):
1215 """
1218 """
1216
1219
1217 Arguments:
1220 Arguments:
1218 path :
1221 path :
1219 startDate :
1222 startDate :
1220 endDate :
1223 endDate :
1221 startTime :
1224 startTime :
1222 endTime :
1225 endTime :
1223 set :
1226 set :
1224 expLabel :
1227 expLabel :
1225 ext :
1228 ext :
1226 online :
1229 online :
1227 delay :
1230 delay :
1228 walk :
1231 walk :
1229 getblock :
1232 getblock :
1230 nTxs :
1233 nTxs :
1231 realtime :
1234 realtime :
1232 blocksize :
1235 blocksize :
1233 blocktime :
1236 blocktime :
1234 skip :
1237 skip :
1235 cursor :
1238 cursor :
1236 warnings :
1239 warnings :
1237 server :
1240 server :
1238 verbose :
1241 verbose :
1239 format :
1242 format :
1240 oneDDict :
1243 oneDDict :
1241 twoDDict :
1244 twoDDict :
1242 independentParam :
1245 independentParam :
1243 """
1246 """
1244
1247
1245 if not(self.isConfig):
1248 if not(self.isConfig):
1246 self.setup(**kwargs)
1249 self.setup(**kwargs)
1247 self.isConfig = True
1250 self.isConfig = True
1248 if self.server is None:
1251 if self.server is None:
1249 self.getData()
1252 self.getData()
1250 else:
1253 else:
1251 self.getFromServer()
1254 self.getFromServer()
1252
1255
1253
1256
1254 class JRODataWriter(Reader):
1257 class JRODataWriter(Reader):
1255
1258
1256 """
1259 """
1257 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1260 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1258 de los datos siempre se realiza por bloques.
1261 de los datos siempre se realiza por bloques.
1259 """
1262 """
1260
1263
1261 setFile = None
1264 setFile = None
1262 profilesPerBlock = None
1265 profilesPerBlock = None
1263 blocksPerFile = None
1266 blocksPerFile = None
1264 nWriteBlocks = 0
1267 nWriteBlocks = 0
1265 fileDate = None
1268 fileDate = None
1266
1269
1267 def __init__(self, dataOut=None):
1270 def __init__(self, dataOut=None):
1268 raise NotImplementedError
1271 raise NotImplementedError
1269
1272
1270 def hasAllDataInBuffer(self):
1273 def hasAllDataInBuffer(self):
1271 raise NotImplementedError
1274 raise NotImplementedError
1272
1275
1273 def setBlockDimension(self):
1276 def setBlockDimension(self):
1274 raise NotImplementedError
1277 raise NotImplementedError
1275
1278
1276 def writeBlock(self):
1279 def writeBlock(self):
1277 raise NotImplementedError
1280 raise NotImplementedError
1278
1281
1279 def putData(self):
1282 def putData(self):
1280 raise NotImplementedError
1283 raise NotImplementedError
1281
1284
1282 def getDtypeWidth(self):
1285 def getDtypeWidth(self):
1283
1286
1284 dtype_index = get_dtype_index(self.dtype)
1287 dtype_index = get_dtype_index(self.dtype)
1285 dtype_width = get_dtype_width(dtype_index)
1288 dtype_width = get_dtype_width(dtype_index)
1286
1289
1287 return dtype_width
1290 return dtype_width
1288
1291
1289 def getProcessFlags(self):
1292 def getProcessFlags(self):
1290
1293
1291 processFlags = 0
1294 processFlags = 0
1292
1295
1293 dtype_index = get_dtype_index(self.dtype)
1296 dtype_index = get_dtype_index(self.dtype)
1294 procflag_dtype = get_procflag_dtype(dtype_index)
1297 procflag_dtype = get_procflag_dtype(dtype_index)
1295
1298
1296 processFlags += procflag_dtype
1299 processFlags += procflag_dtype
1297
1300
1298 if self.dataOut.flagDecodeData:
1301 if self.dataOut.flagDecodeData:
1299 processFlags += PROCFLAG.DECODE_DATA
1302 processFlags += PROCFLAG.DECODE_DATA
1300
1303
1301 if self.dataOut.flagDeflipData:
1304 if self.dataOut.flagDeflipData:
1302 processFlags += PROCFLAG.DEFLIP_DATA
1305 processFlags += PROCFLAG.DEFLIP_DATA
1303
1306
1304 if self.dataOut.code is not None:
1307 if self.dataOut.code is not None:
1305 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1308 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1306
1309
1307 if self.dataOut.nCohInt > 1:
1310 if self.dataOut.nCohInt > 1:
1308 processFlags += PROCFLAG.COHERENT_INTEGRATION
1311 processFlags += PROCFLAG.COHERENT_INTEGRATION
1309
1312
1310 if self.dataOut.type == "Spectra":
1313 if self.dataOut.type == "Spectra":
1311 if self.dataOut.nIncohInt > 1:
1314 if self.dataOut.nIncohInt > 1:
1312 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1315 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1313
1316
1314 if self.dataOut.data_dc is not None:
1317 if self.dataOut.data_dc is not None:
1315 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1318 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1316
1319
1317 if self.dataOut.flagShiftFFT:
1320 if self.dataOut.flagShiftFFT:
1318 processFlags += PROCFLAG.SHIFT_FFT_DATA
1321 processFlags += PROCFLAG.SHIFT_FFT_DATA
1319
1322
1320 return processFlags
1323 return processFlags
1321
1324
1322 def setBasicHeader(self):
1325 def setBasicHeader(self):
1323
1326
1324 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1327 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1325 self.basicHeaderObj.version = self.versionFile
1328 self.basicHeaderObj.version = self.versionFile
1326 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1329 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1327 utc = numpy.floor(self.dataOut.utctime)
1330 utc = numpy.floor(self.dataOut.utctime)
1328 milisecond = (self.dataOut.utctime - utc) * 1000.0
1331 milisecond = (self.dataOut.utctime - utc) * 1000.0
1329 self.basicHeaderObj.utc = utc
1332 self.basicHeaderObj.utc = utc
1330 self.basicHeaderObj.miliSecond = milisecond
1333 self.basicHeaderObj.miliSecond = milisecond
1331 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1334 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1332 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1335 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1333 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1336 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1334
1337
1335 def setFirstHeader(self):
1338 def setFirstHeader(self):
1336 """
1339 """
1337 Obtiene una copia del First Header
1340 Obtiene una copia del First Header
1338
1341
1339 Affected:
1342 Affected:
1340
1343
1341 self.basicHeaderObj
1344 self.basicHeaderObj
1342 self.systemHeaderObj
1345 self.systemHeaderObj
1343 self.radarControllerHeaderObj
1346 self.radarControllerHeaderObj
1344 self.processingHeaderObj self.
1347 self.processingHeaderObj self.
1345
1348
1346 Return:
1349 Return:
1347 None
1350 None
1348 """
1351 """
1349
1352
1350 raise NotImplementedError
1353 raise NotImplementedError
1351
1354
1352 def __writeFirstHeader(self):
1355 def __writeFirstHeader(self):
1353 """
1356 """
1354 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1357 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1355
1358
1356 Affected:
1359 Affected:
1357 __dataType
1360 __dataType
1358
1361
1359 Return:
1362 Return:
1360 None
1363 None
1361 """
1364 """
1362
1365
1363 # CALCULAR PARAMETROS
1366 # CALCULAR PARAMETROS
1364
1367
1365 sizeLongHeader = self.systemHeaderObj.size + \
1368 sizeLongHeader = self.systemHeaderObj.size + \
1366 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1369 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1367 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1370 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1368
1371
1369 self.basicHeaderObj.write(self.fp)
1372 self.basicHeaderObj.write(self.fp)
1370 self.systemHeaderObj.write(self.fp)
1373 self.systemHeaderObj.write(self.fp)
1371 self.radarControllerHeaderObj.write(self.fp)
1374 self.radarControllerHeaderObj.write(self.fp)
1372 self.processingHeaderObj.write(self.fp)
1375 self.processingHeaderObj.write(self.fp)
1373
1376
1374 def __setNewBlock(self):
1377 def __setNewBlock(self):
1375 """
1378 """
1376 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1379 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1377
1380
1378 Return:
1381 Return:
1379 0 : si no pudo escribir nada
1382 0 : si no pudo escribir nada
1380 1 : Si escribio el Basic el First Header
1383 1 : Si escribio el Basic el First Header
1381 """
1384 """
1382 if self.fp == None:
1385 if self.fp == None:
1383 self.setNextFile()
1386 self.setNextFile()
1384
1387
1385 if self.flagIsNewFile:
1388 if self.flagIsNewFile:
1386 return 1
1389 return 1
1387
1390
1388 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1391 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1389 self.basicHeaderObj.write(self.fp)
1392 self.basicHeaderObj.write(self.fp)
1390 return 1
1393 return 1
1391
1394
1392 if not(self.setNextFile()):
1395 if not(self.setNextFile()):
1393 return 0
1396 return 0
1394
1397
1395 return 1
1398 return 1
1396
1399
1397 def writeNextBlock(self):
1400 def writeNextBlock(self):
1398 """
1401 """
1399 Selecciona el bloque siguiente de datos y los escribe en un file
1402 Selecciona el bloque siguiente de datos y los escribe en un file
1400
1403
1401 Return:
1404 Return:
1402 0 : Si no hizo pudo escribir el bloque de datos
1405 0 : Si no hizo pudo escribir el bloque de datos
1403 1 : Si no pudo escribir el bloque de datos
1406 1 : Si no pudo escribir el bloque de datos
1404 """
1407 """
1405 if not(self.__setNewBlock()):
1408 if not(self.__setNewBlock()):
1406 return 0
1409 return 0
1407
1410
1408 self.writeBlock()
1411 self.writeBlock()
1409
1412
1410 print("[Writing] Block No. %d/%d" % (self.blockIndex,
1413 print("[Writing] Block No. %d/%d" % (self.blockIndex,
1411 self.processingHeaderObj.dataBlocksPerFile))
1414 self.processingHeaderObj.dataBlocksPerFile))
1412
1415
1413 return 1
1416 return 1
1414
1417
1415 def setNextFile(self):
1418 def setNextFile(self):
1416 """Determina el siguiente file que sera escrito
1419 """Determina el siguiente file que sera escrito
1417
1420
1418 Affected:
1421 Affected:
1419 self.filename
1422 self.filename
1420 self.subfolder
1423 self.subfolder
1421 self.fp
1424 self.fp
1422 self.setFile
1425 self.setFile
1423 self.flagIsNewFile
1426 self.flagIsNewFile
1424
1427
1425 Return:
1428 Return:
1426 0 : Si el archivo no puede ser escrito
1429 0 : Si el archivo no puede ser escrito
1427 1 : Si el archivo esta listo para ser escrito
1430 1 : Si el archivo esta listo para ser escrito
1428 """
1431 """
1429 ext = self.ext
1432 ext = self.ext
1430 path = self.path
1433 path = self.path
1431
1434
1432 if self.fp != None:
1435 if self.fp != None:
1433 self.fp.close()
1436 self.fp.close()
1434
1437
1435
1438
1436 if not os.path.exists(path):
1439 if not os.path.exists(path):
1437 os.mkdir(path)
1440 os.mkdir(path)
1438
1441
1439 timeTuple = time.localtime(self.dataOut.utctime)
1442 timeTuple = time.localtime(self.dataOut.utctime)
1440 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1443 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1441
1444
1442 fullpath = os.path.join(path, subfolder)
1445 fullpath = os.path.join(path, subfolder)
1443 setFile = self.setFile
1446 setFile = self.setFile
1444
1447
1445 if not(os.path.exists(fullpath)):
1448 if not(os.path.exists(fullpath)):
1446 os.mkdir(fullpath)
1449 os.mkdir(fullpath)
1447 setFile = -1 # inicializo mi contador de seteo
1450 setFile = -1 # inicializo mi contador de seteo
1448 else:
1451 else:
1449 filesList = os.listdir(fullpath)
1452 filesList = os.listdir(fullpath)
1450 if len(filesList) > 0:
1453 if len(filesList) > 0:
1451 filesList = sorted(filesList, key=str.lower)
1454 filesList = sorted(filesList, key=str.lower)
1452 filen = filesList[-1]
1455 filen = filesList[-1]
1453 # el filename debera tener el siguiente formato
1456 # el filename debera tener el siguiente formato
1454 # 0 1234 567 89A BCDE (hex)
1457 # 0 1234 567 89A BCDE (hex)
1455 # x YYYY DDD SSS .ext
1458 # x YYYY DDD SSS .ext
1456 if isNumber(filen[8:11]):
1459 if isNumber(filen[8:11]):
1457 # inicializo mi contador de seteo al seteo del ultimo file
1460 # inicializo mi contador de seteo al seteo del ultimo file
1458 setFile = int(filen[8:11])
1461 setFile = int(filen[8:11])
1459 else:
1462 else:
1460 setFile = -1
1463 setFile = -1
1461 else:
1464 else:
1462 setFile = -1 # inicializo mi contador de seteo
1465 setFile = -1 # inicializo mi contador de seteo
1463
1466
1464 setFile += 1
1467 setFile += 1
1465
1468
1466 # If this is a new day it resets some values
1469 # If this is a new day it resets some values
1467 if self.dataOut.datatime.date() > self.fileDate:
1470 if self.dataOut.datatime.date() > self.fileDate:
1468 setFile = 0
1471 setFile = 0
1469 self.nTotalBlocks = 0
1472 self.nTotalBlocks = 0
1470
1473
1471 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1474 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1472 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1475 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1473
1476
1474 filename = os.path.join(path, subfolder, filen)
1477 filename = os.path.join(path, subfolder, filen)
1475
1478
1476 fp = open(filename, 'wb')
1479 fp = open(filename, 'wb')
1477
1480
1478 self.blockIndex = 0
1481 self.blockIndex = 0
1479 self.filename = filename
1482 self.filename = filename
1480 self.subfolder = subfolder
1483 self.subfolder = subfolder
1481 self.fp = fp
1484 self.fp = fp
1482 self.setFile = setFile
1485 self.setFile = setFile
1483 self.flagIsNewFile = 1
1486 self.flagIsNewFile = 1
1484 self.fileDate = self.dataOut.datatime.date()
1487 self.fileDate = self.dataOut.datatime.date()
1485 self.setFirstHeader()
1488 self.setFirstHeader()
1486
1489
1487 print('[Writing] Opening file: %s' % self.filename)
1490 print('[Writing] Opening file: %s' % self.filename)
1488
1491
1489 self.__writeFirstHeader()
1492 self.__writeFirstHeader()
1490
1493
1491 return 1
1494 return 1
1492
1495
1493 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1496 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1494 """
1497 """
1495 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1498 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1496
1499
1497 Inputs:
1500 Inputs:
1498 path : directory where data will be saved
1501 path : directory where data will be saved
1499 profilesPerBlock : number of profiles per block
1502 profilesPerBlock : number of profiles per block
1500 set : initial file set
1503 set : initial file set
1501 datatype : An integer number that defines data type:
1504 datatype : An integer number that defines data type:
1502 0 : int8 (1 byte)
1505 0 : int8 (1 byte)
1503 1 : int16 (2 bytes)
1506 1 : int16 (2 bytes)
1504 2 : int32 (4 bytes)
1507 2 : int32 (4 bytes)
1505 3 : int64 (8 bytes)
1508 3 : int64 (8 bytes)
1506 4 : float32 (4 bytes)
1509 4 : float32 (4 bytes)
1507 5 : double64 (8 bytes)
1510 5 : double64 (8 bytes)
1508
1511
1509 Return:
1512 Return:
1510 0 : Si no realizo un buen seteo
1513 0 : Si no realizo un buen seteo
1511 1 : Si realizo un buen seteo
1514 1 : Si realizo un buen seteo
1512 """
1515 """
1513
1516
1514 if ext == None:
1517 if ext == None:
1515 ext = self.ext
1518 ext = self.ext
1516
1519
1517 self.ext = ext.lower()
1520 self.ext = ext.lower()
1518
1521
1519 self.path = path
1522 self.path = path
1520
1523
1521 if set is None:
1524 if set is None:
1522 self.setFile = -1
1525 self.setFile = -1
1523 else:
1526 else:
1524 self.setFile = set - 1
1527 self.setFile = set - 1
1525
1528
1526 self.blocksPerFile = blocksPerFile
1529 self.blocksPerFile = blocksPerFile
1527 self.profilesPerBlock = profilesPerBlock
1530 self.profilesPerBlock = profilesPerBlock
1528 self.dataOut = dataOut
1531 self.dataOut = dataOut
1529 self.fileDate = self.dataOut.datatime.date()
1532 self.fileDate = self.dataOut.datatime.date()
1530 self.dtype = self.dataOut.dtype
1533 self.dtype = self.dataOut.dtype
1531
1534
1532 if datatype is not None:
1535 if datatype is not None:
1533 self.dtype = get_numpy_dtype(datatype)
1536 self.dtype = get_numpy_dtype(datatype)
1534
1537
1535 if not(self.setNextFile()):
1538 if not(self.setNextFile()):
1536 print("[Writing] There isn't a next file")
1539 print("[Writing] There isn't a next file")
1537 return 0
1540 return 0
1538
1541
1539 self.setBlockDimension()
1542 self.setBlockDimension()
1540
1543
1541 return 1
1544 return 1
1542
1545
1543 def run(self, dataOut, path, blocksPerFile=100, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1546 def run(self, dataOut, path, blocksPerFile=100, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1544
1547
1545 if not(self.isConfig):
1548 if not(self.isConfig):
1546
1549
1547 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1550 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1548 set=set, ext=ext, datatype=datatype, **kwargs)
1551 set=set, ext=ext, datatype=datatype, **kwargs)
1549 self.isConfig = True
1552 self.isConfig = True
1550
1553
1551 self.dataOut = dataOut
1554 self.dataOut = dataOut
1552 self.putData()
1555 self.putData()
1553 return self.dataOut
1556 return self.dataOut
1554
1557
1555 @MPDecorator
1558 @MPDecorator
1556 class printInfo(Operation):
1559 class printInfo(Operation):
1557
1560
1558 def __init__(self):
1561 def __init__(self):
1559
1562
1560 Operation.__init__(self)
1563 Operation.__init__(self)
1561 self.__printInfo = True
1564 self.__printInfo = True
1562
1565
1563 def run(self, dataOut, headers = ['systemHeaderObj', 'radarControllerHeaderObj', 'processingHeaderObj']):
1566 def run(self, dataOut, headers = ['systemHeaderObj', 'radarControllerHeaderObj', 'processingHeaderObj']):
1564 if self.__printInfo == False:
1567 if self.__printInfo == False:
1565 return
1568 return
1566
1569
1567 for header in headers:
1570 for header in headers:
1568 if hasattr(dataOut, header):
1571 if hasattr(dataOut, header):
1569 obj = getattr(dataOut, header)
1572 obj = getattr(dataOut, header)
1570 if hasattr(obj, 'printInfo'):
1573 if hasattr(obj, 'printInfo'):
1571 obj.printInfo()
1574 obj.printInfo()
1572 else:
1575 else:
1573 print(obj)
1576 print(obj)
1574 else:
1577 else:
1575 log.warning('Header {} Not found in object'.format(header))
1578 log.warning('Header {} Not found in object'.format(header))
1576
1579
1577 self.__printInfo = False
1580 self.__printInfo = False
Show file before | Show file after | Hide comments
@@ -1,665 +1,665
1 ''''
1 ''''
2 Created on Set 9, 2015
2 Created on Set 9, 2015
3
3
4 @author: roj-idl71 Karim Kuyeng
4 @author: roj-idl71 Karim Kuyeng
5
5
6 @update: 2021, Joab Apaza
6 @update: 2021, Joab Apaza
7 '''
7 '''
8
8
9 import os
9 import os
10 import sys
10 import sys
11 import glob
11 import glob
12 import fnmatch
12 import fnmatch
13 import datetime
13 import datetime
14 import time
14 import time
15 import re
15 import re
16 import h5py
16 import h5py
17 import numpy
17 import numpy
18
18
19 try:
19 try:
20 from gevent import sleep
20 from gevent import sleep
21 except:
21 except:
22 from time import sleep
22 from time import sleep
23
23
24 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
24 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
25 from schainpy.model.data.jrodata import Voltage
25 from schainpy.model.data.jrodata import Voltage
26 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
26 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
27 from numpy import imag
27 from numpy import imag
28 from schainpy.utils import log
28 from schainpy.utils import log
29
29
30
30
31 class AMISRReader(ProcessingUnit):
31 class AMISRReader(ProcessingUnit):
32 '''
32 '''
33 classdocs
33 classdocs
34 '''
34 '''
35
35
36 def __init__(self):
36 def __init__(self):
37 '''
37 '''
38 Constructor
38 Constructor
39 '''
39 '''
40
40
41 ProcessingUnit.__init__(self)
41 ProcessingUnit.__init__(self)
42
42
43 self.set = None
43 self.set = None
44 self.subset = None
44 self.subset = None
45 self.extension_file = '.h5'
45 self.extension_file = '.h5'
46 self.dtc_str = 'dtc'
46 self.dtc_str = 'dtc'
47 self.dtc_id = 0
47 self.dtc_id = 0
48 self.status = True
48 self.status = True
49 self.isConfig = False
49 self.isConfig = False
50 self.dirnameList = []
50 self.dirnameList = []
51 self.filenameList = []
51 self.filenameList = []
52 self.fileIndex = None
52 self.fileIndex = None
53 self.flagNoMoreFiles = False
53 self.flagNoMoreFiles = False
54 self.flagIsNewFile = 0
54 self.flagIsNewFile = 0
55 self.filename = ''
55 self.filename = ''
56 self.amisrFilePointer = None
56 self.amisrFilePointer = None
57 self.realBeamCode = []
57 self.realBeamCode = []
58 self.beamCodeMap = None
58 self.beamCodeMap = None
59 self.azimuthList = []
59 self.azimuthList = []
60 self.elevationList = []
60 self.elevationList = []
61 self.dataShape = None
61 self.dataShape = None
62 self.flag_old_beams = False
62 self.flag_old_beams = False
63
63
64
64
65 self.profileIndex = 0
65 self.profileIndex = 0
66
66
67
67
68 self.beamCodeByFrame = None
68 self.beamCodeByFrame = None
69 self.radacTimeByFrame = None
69 self.radacTimeByFrame = None
70
70
71 self.dataset = None
71 self.dataset = None
72
72
73 self.__firstFile = True
73 self.__firstFile = True
74
74
75 self.buffer = None
75 self.buffer = None
76
76
77 self.timezone = 'ut'
77 self.timezone = 'ut'
78
78
79 self.__waitForNewFile = 20
79 self.__waitForNewFile = 20
80 self.__filename_online = None
80 self.__filename_online = None
81 #Is really necessary create the output object in the initializer
81 #Is really necessary create the output object in the initializer
82 self.dataOut = Voltage()
82 self.dataOut = Voltage()
83 self.dataOut.error=False
83 self.dataOut.error=False
84
84 self.margin_days = 1
85
85
86 def setup(self,path=None,
86 def setup(self,path=None,
87 startDate=None,
87 startDate=None,
88 endDate=None,
88 endDate=None,
89 startTime=None,
89 startTime=None,
90 endTime=None,
90 endTime=None,
91 walk=True,
91 walk=True,
92 timezone='ut',
92 timezone='ut',
93 all=0,
93 all=0,
94 code = None,
94 code = None,
95 nCode = 0,
95 nCode = 0,
96 nBaud = 0,
96 nBaud = 0,
97 online=False,
97 online=False,
98 old_beams=False):
98 old_beams=False,
99 margin_days=1):
99
100
100
101
101
102
102 self.timezone = timezone
103 self.timezone = timezone
103 self.all = all
104 self.all = all
104 self.online = online
105 self.online = online
105 self.flag_old_beams = old_beams
106 self.flag_old_beams = old_beams
106 self.code = code
107 self.code = code
107 self.nCode = int(nCode)
108 self.nCode = int(nCode)
108 self.nBaud = int(nBaud)
109 self.nBaud = int(nBaud)
109
110 self.margin_days = margin_days
110
111
111
112
112 #self.findFiles()
113 #self.findFiles()
113 if not(online):
114 if not(online):
114 #Busqueda de archivos offline
115 #Busqueda de archivos offline
115 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
116 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
116 else:
117 else:
117 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
118 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
118
119
119 if not(self.filenameList):
120 if not(self.filenameList):
120 raise schainpy.admin.SchainWarning("There is no files into the folder: %s"%(path))
121 raise schainpy.admin.SchainWarning("There is no files into the folder: %s"%(path))
121 sys.exit()
122 sys.exit()
122
123
123 self.fileIndex = 0
124 self.fileIndex = 0
124
125
125 self.readNextFile(online)
126 self.readNextFile(online)
126
127
127 '''
128 '''
128 Add code
129 Add code
129 '''
130 '''
130 self.isConfig = True
131 self.isConfig = True
131 # print("Setup Done")
132 # print("Setup Done")
132 pass
133 pass
133
134
134
135
135 def readAMISRHeader(self,fp):
136 def readAMISRHeader(self,fp):
136
137
137 if self.isConfig and (not self.flagNoMoreFiles):
138 if self.isConfig and (not self.flagNoMoreFiles):
138 newShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
139 newShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
139 if self.dataShape != newShape and newShape != None:
140 if self.dataShape != newShape and newShape != None:
140 raise schainpy.admin.SchainError("NEW FILE HAS A DIFFERENT SHAPE: ")
141 raise schainpy.admin.SchainError("NEW FILE HAS A DIFFERENT SHAPE: ")
141 print(self.dataShape,newShape,"\n")
142 print(self.dataShape,newShape,"\n")
142 return 0
143 return 0
143 else:
144 else:
144 self.dataShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
145 self.dataShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
145
146
146
147
147 header = 'Raw11/Data/RadacHeader'
148 header = 'Raw11/Data/RadacHeader'
148 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
149 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
149 if (self.startDate> datetime.date(2021, 7, 15)) or self.flag_old_beams: #Se cambiΓ³ la forma de extracciΓ³n de Apuntes el 17 o forzar con flag de reorganizaciΓ³n
150 if (self.startDate> datetime.date(2021, 7, 15)) or self.flag_old_beams: #Se cambiΓ³ la forma de extracciΓ³n de Apuntes el 17 o forzar con flag de reorganizaciΓ³n
150 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
151 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
151 self.trueBeams = self.beamcodeFile.split("\n")
152 self.trueBeams = self.beamcodeFile.split("\n")
152 self.trueBeams.pop()#remove last
153 self.trueBeams.pop()#remove last
153 [self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
154 [self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
154 self.beamCode = [int(x, 16) for x in self.realBeamCode]
155 self.beamCode = [int(x, 16) for x in self.realBeamCode]
155 else:
156 else:
156 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
157 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
157 self.beamCode = _beamCode[0,:]
158 self.beamCode = _beamCode[0,:]
158
159
159 if self.beamCodeMap == None:
160 if self.beamCodeMap == None:
160 self.beamCodeMap = fp['Setup/BeamcodeMap']
161 self.beamCodeMap = fp['Setup/BeamcodeMap']
161 for beam in self.beamCode:
162 for beam in self.beamCode:
162 beamAziElev = numpy.where(self.beamCodeMap[:,0]==beam)
163 beamAziElev = numpy.where(self.beamCodeMap[:,0]==beam)
163 beamAziElev = beamAziElev[0].squeeze()
164 beamAziElev = beamAziElev[0].squeeze()
164 self.azimuthList.append(self.beamCodeMap[beamAziElev,1])
165 self.azimuthList.append(self.beamCodeMap[beamAziElev,1])
165 self.elevationList.append(self.beamCodeMap[beamAziElev,2])
166 self.elevationList.append(self.beamCodeMap[beamAziElev,2])
166 #print("Beamssss: ",self.beamCodeMap[beamAziElev,1],self.beamCodeMap[beamAziElev,2])
167 #print("Beamssss: ",self.beamCodeMap[beamAziElev,1],self.beamCodeMap[beamAziElev,2])
167 #print(self.beamCode)
168 #print(self.beamCode)
168 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
169 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
169 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
170 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
170 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
171 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
171 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
172 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
172 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
173 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
173 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
174 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
174 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
175 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
175 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
176 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
176 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
177 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
177 self.frequency = fp.get('Rx/Frequency')
178 self.frequency = fp.get('Rx/Frequency')
178 txAus = fp.get('Raw11/Data/Pulsewidth')
179 txAus = fp.get('Raw11/Data/Pulsewidth')
179
180
180
181
181 self.nblocks = self.pulseCount.shape[0] #nblocks
182 self.nblocks = self.pulseCount.shape[0] #nblocks
182
183
183 self.nprofiles = self.pulseCount.shape[1] #nprofile
184 self.nprofiles = self.pulseCount.shape[1] #nprofile
184 self.nsa = self.nsamplesPulse[0,0] #ngates
185 self.nsa = self.nsamplesPulse[0,0] #ngates
185 self.nchannels = len(self.beamCode)
186 self.nchannels = len(self.beamCode)
186 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
187 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
187 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
188 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
188 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
189 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
189
190
190 #filling radar controller header parameters
191 #filling radar controller header parameters
191 self.__ippKm = self.ippSeconds *.15*1e6 # in km
192 self.__ippKm = self.ippSeconds *.15*1e6 # in km
192 self.__txA = (txAus[()])*.15 #(ipp[us]*.15km/1us) in km
193 self.__txA = (txAus[()])*.15 #(ipp[us]*.15km/1us) in km
193 self.__txB = 0
194 self.__txB = 0
194 nWindows=1
195 nWindows=1
195 self.__nSamples = self.nsa
196 self.__nSamples = self.nsa
196 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
197 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
197 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
198 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
198
199 #print("amisr-ipp:",self.ippSeconds, self.__ippKm)
199 #for now until understand why the code saved is different (code included even though code not in tuf file)
200 #for now until understand why the code saved is different (code included even though code not in tuf file)
200 #self.__codeType = 0
201 #self.__codeType = 0
201 # self.__nCode = None
202 # self.__nCode = None
202 # self.__nBaud = None
203 # self.__nBaud = None
203 self.__code = self.code
204 self.__code = self.code
204 self.__codeType = 0
205 self.__codeType = 0
205 if self.code != None:
206 if self.code != None:
206 self.__codeType = 1
207 self.__codeType = 1
207 self.__nCode = self.nCode
208 self.__nCode = self.nCode
208 self.__nBaud = self.nBaud
209 self.__nBaud = self.nBaud
209 #self.__code = 0
210 #self.__code = 0
210
211
211 #filling system header parameters
212 #filling system header parameters
212 self.__nSamples = self.nsa
213 self.__nSamples = self.nsa
213 self.newProfiles = self.nprofiles/self.nchannels
214 self.newProfiles = self.nprofiles/self.nchannels
214 self.__channelList = list(range(self.nchannels))
215 self.__channelList = list(range(self.nchannels))
215
216
216 self.__frequency = self.frequency[0][0]
217 self.__frequency = self.frequency[0][0]
217
218
218
219
219 return 1
220 return 1
220
221
221
222
222 def createBuffers(self):
223 def createBuffers(self):
223
224
224 pass
225 pass
225
226
226 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
227 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
227 self.path = path
228 self.path = path
228 self.startDate = startDate
229 self.startDate = startDate
229 self.endDate = endDate
230 self.endDate = endDate
230 self.startTime = startTime
231 self.startTime = startTime
231 self.endTime = endTime
232 self.endTime = endTime
232 self.walk = walk
233 self.walk = walk
233
234
234 def __checkPath(self):
235 def __checkPath(self):
235 if os.path.exists(self.path):
236 if os.path.exists(self.path):
236 self.status = 1
237 self.status = 1
237 else:
238 else:
238 self.status = 0
239 self.status = 0
239 print('Path:%s does not exists'%self.path)
240 print('Path:%s does not exists'%self.path)
240
241
241 return
242 return
242
243
243
244
244 def __selDates(self, amisr_dirname_format):
245 def __selDates(self, amisr_dirname_format):
245 try:
246 try:
246 year = int(amisr_dirname_format[0:4])
247 year = int(amisr_dirname_format[0:4])
247 month = int(amisr_dirname_format[4:6])
248 month = int(amisr_dirname_format[4:6])
248 dom = int(amisr_dirname_format[6:8])
249 dom = int(amisr_dirname_format[6:8])
249 thisDate = datetime.date(year,month,dom)
250 thisDate = datetime.date(year,month,dom)
250 #margen de un dΓ­a extra, igual luego se filtra for fecha y hora
251 #margen de un dΓ­a extra, igual luego se filtra for fecha y hora
251 if (thisDate>=(self.startDate - datetime.timedelta(days=1)) and thisDate <= (self.endDate)+ datetime.timedelta(days=1)):
252 if (thisDate>=(self.startDate - datetime.timedelta(days=self.margin_days)) and thisDate <= (self.endDate)+ datetime.timedelta(days=1)):
252 return amisr_dirname_format
253 return amisr_dirname_format
253 except:
254 except:
254 return None
255 return None
255
256
256
257
257 def __findDataForDates(self,online=False):
258 def __findDataForDates(self,online=False):
258
259
259 if not(self.status):
260 if not(self.status):
260 return None
261 return None
261
262
262 pat = '\d+.\d+'
263 pat = '\d+.\d+'
263 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
264 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
264 dirnameList = [x for x in dirnameList if x!=None]
265 dirnameList = [x for x in dirnameList if x!=None]
265 dirnameList = [x.string for x in dirnameList]
266 dirnameList = [x.string for x in dirnameList]
266 if not(online):
267 if not(online):
267 dirnameList = [self.__selDates(x) for x in dirnameList]
268 dirnameList = [self.__selDates(x) for x in dirnameList]
268 dirnameList = [x for x in dirnameList if x!=None]
269 dirnameList = [x for x in dirnameList if x!=None]
269 if len(dirnameList)>0:
270 if len(dirnameList)>0:
270 self.status = 1
271 self.status = 1
271 self.dirnameList = dirnameList
272 self.dirnameList = dirnameList
272 self.dirnameList.sort()
273 self.dirnameList.sort()
273 else:
274 else:
274 self.status = 0
275 self.status = 0
275 return None
276 return None
276
277
277 def __getTimeFromData(self):
278 def __getTimeFromData(self):
278 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
279 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
279 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
280 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
280
281
281 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
282 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
282 print('........................................')
283 print('........................................')
283 filter_filenameList = []
284 filter_filenameList = []
284 self.filenameList.sort()
285 self.filenameList.sort()
285 total_files = len(self.filenameList)
286 total_files = len(self.filenameList)
286 #for i in range(len(self.filenameList)-1):
287 #for i in range(len(self.filenameList)-1):
287 for i in range(total_files):
288 for i in range(total_files):
288 filename = self.filenameList[i]
289 filename = self.filenameList[i]
289 #print("file-> ",filename)
290 #print("file-> ",filename)
290 try:
291 try:
291 fp = h5py.File(filename,'r')
292 fp = h5py.File(filename,'r')
292 time_str = fp.get('Time/RadacTimeString')
293 time_str = fp.get('Time/RadacTimeString')
293
294
294 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
295 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
295 #startDateTimeStr_File = "2019-12-16 09:21:11"
296 #startDateTimeStr_File = "2019-12-16 09:21:11"
296 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
297 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
297 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
298 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
298
299
299 #endDateTimeStr_File = "2019-12-16 11:10:11"
300 #endDateTimeStr_File = "2019-12-16 11:10:11"
300 endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
301 endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
301 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
302 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
302 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
303 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
303
304
304 fp.close()
305 fp.close()
305
306
306 #print("check time", startDateTime_File)
307 #print("check time", startDateTime_File)
307 if self.timezone == 'lt':
308 if self.timezone == 'lt':
308 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
309 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
309 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
310 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
310 if (startDateTime_File >=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
311 if (startDateTime_File >=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
311 filter_filenameList.append(filename)
312 filter_filenameList.append(filename)
312
313
313 if (startDateTime_File>endDateTime_Reader):
314 if (startDateTime_File>endDateTime_Reader):
314 break
315 break
315 except Exception as e:
316 except Exception as e:
316 log.warning("Error opening file {} -> {}".format(os.path.split(filename)[1],e))
317 log.warning("Error opening file {} -> {}".format(os.path.split(filename)[1],e))
317
318
318 filter_filenameList.sort()
319 filter_filenameList.sort()
319 self.filenameList = filter_filenameList
320 self.filenameList = filter_filenameList
320
321
321 return 1
322 return 1
322
323
323 def __filterByGlob1(self, dirName):
324 def __filterByGlob1(self, dirName):
324 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
325 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
325 filter_files.sort()
326 filter_files.sort()
326 filterDict = {}
327 filterDict = {}
327 filterDict.setdefault(dirName)
328 filterDict.setdefault(dirName)
328 filterDict[dirName] = filter_files
329 filterDict[dirName] = filter_files
329 return filterDict
330 return filterDict
330
331
331 def __getFilenameList(self, fileListInKeys, dirList):
332 def __getFilenameList(self, fileListInKeys, dirList):
332 for value in fileListInKeys:
333 for value in fileListInKeys:
333 dirName = list(value.keys())[0]
334 dirName = list(value.keys())[0]
334 for file in value[dirName]:
335 for file in value[dirName]:
335 filename = os.path.join(dirName, file)
336 filename = os.path.join(dirName, file)
336 self.filenameList.append(filename)
337 self.filenameList.append(filename)
337
338
338
339
339 def __selectDataForTimes(self, online=False):
340 def __selectDataForTimes(self, online=False):
340 #aun no esta implementado el filtro for tiempo
341 #aun no esta implementado el filtro for tiempo
341 if not(self.status):
342 if not(self.status):
342 return None
343 return None
343
344
344 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
345 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
345 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
346 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
346 self.__getFilenameList(fileListInKeys, dirList)
347 self.__getFilenameList(fileListInKeys, dirList)
347 if not(online):
348 if not(online):
348 #filtro por tiempo
349 #filtro por tiempo
349 if not(self.all):
350 if not(self.all):
350 self.__getTimeFromData()
351 self.__getTimeFromData()
351
352
352 if len(self.filenameList)>0:
353 if len(self.filenameList)>0:
353 self.status = 1
354 self.status = 1
354 self.filenameList.sort()
355 self.filenameList.sort()
355 else:
356 else:
356 self.status = 0
357 self.status = 0
357 return None
358 return None
358
359
359 else:
360 else:
360 #get the last file - 1
361 #get the last file - 1
361 self.filenameList = [self.filenameList[-2]]
362 self.filenameList = [self.filenameList[-2]]
362 new_dirnameList = []
363 new_dirnameList = []
363 for dirname in self.dirnameList:
364 for dirname in self.dirnameList:
364 junk = numpy.array([dirname in x for x in self.filenameList])
365 junk = numpy.array([dirname in x for x in self.filenameList])
365 junk_sum = junk.sum()
366 junk_sum = junk.sum()
366 if junk_sum > 0:
367 if junk_sum > 0:
367 new_dirnameList.append(dirname)
368 new_dirnameList.append(dirname)
368 self.dirnameList = new_dirnameList
369 self.dirnameList = new_dirnameList
369 return 1
370 return 1
370
371
371 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
372 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
372 endTime=datetime.time(23,59,59),walk=True):
373 endTime=datetime.time(23,59,59),walk=True):
373
374
374 if endDate ==None:
375 if endDate ==None:
375 startDate = datetime.datetime.utcnow().date()
376 startDate = datetime.datetime.utcnow().date()
376 endDate = datetime.datetime.utcnow().date()
377 endDate = datetime.datetime.utcnow().date()
377
378
378 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
379 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
379
380
380 self.__checkPath()
381 self.__checkPath()
381
382
382 self.__findDataForDates(online=True)
383 self.__findDataForDates(online=True)
383
384
384 self.dirnameList = [self.dirnameList[-1]]
385 self.dirnameList = [self.dirnameList[-1]]
385
386
386 self.__selectDataForTimes(online=True)
387 self.__selectDataForTimes(online=True)
387
388
388 return
389 return
389
390
390
391
391 def searchFilesOffLine(self,
392 def searchFilesOffLine(self,
392 path,
393 path,
393 startDate,
394 startDate,
394 endDate,
395 endDate,
395 startTime=datetime.time(0,0,0),
396 startTime=datetime.time(0,0,0),
396 endTime=datetime.time(23,59,59),
397 endTime=datetime.time(23,59,59),
397 walk=True):
398 walk=True):
398
399
399 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
400 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
400
401
401 self.__checkPath()
402 self.__checkPath()
402
403
403 self.__findDataForDates()
404 self.__findDataForDates()
404
405
405 self.__selectDataForTimes()
406 self.__selectDataForTimes()
406
407
407 for i in range(len(self.filenameList)):
408 for i in range(len(self.filenameList)):
408 print("%s" %(self.filenameList[i]))
409 print("%s" %(self.filenameList[i]))
409
410
410 return
411 return
411
412
412 def __setNextFileOffline(self):
413 def __setNextFileOffline(self):
413
414
414 try:
415 try:
415 self.filename = self.filenameList[self.fileIndex]
416 self.filename = self.filenameList[self.fileIndex]
416 self.amisrFilePointer = h5py.File(self.filename,'r')
417 self.amisrFilePointer = h5py.File(self.filename,'r')
417 self.fileIndex += 1
418 self.fileIndex += 1
418 except:
419 except:
419 self.flagNoMoreFiles = 1
420 self.flagNoMoreFiles = 1
420 raise schainpy.admin.SchainError('No more files to read')
421 raise schainpy.admin.SchainError('No more files to read')
421 return 0
422 return 0
422
423
423 self.flagIsNewFile = 1
424 self.flagIsNewFile = 1
424 print("Setting the file: %s"%self.filename)
425 print("Setting the file: %s"%self.filename)
425
426
426 return 1
427 return 1
427
428
428
429
429 def __setNextFileOnline(self):
430 def __setNextFileOnline(self):
430 filename = self.filenameList[0]
431 filename = self.filenameList[0]
431 if self.__filename_online != None:
432 if self.__filename_online != None:
432 self.__selectDataForTimes(online=True)
433 self.__selectDataForTimes(online=True)
433 filename = self.filenameList[0]
434 filename = self.filenameList[0]
434 wait = 0
435 wait = 0
435 self.__waitForNewFile=300 ## DEBUG:
436 self.__waitForNewFile=300 ## DEBUG:
436 while self.__filename_online == filename:
437 while self.__filename_online == filename:
437 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
438 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
438 if wait == 5:
439 if wait == 5:
439 self.flagNoMoreFiles = 1
440 self.flagNoMoreFiles = 1
440 return 0
441 return 0
441 sleep(self.__waitForNewFile)
442 sleep(self.__waitForNewFile)
442 self.__selectDataForTimes(online=True)
443 self.__selectDataForTimes(online=True)
443 filename = self.filenameList[0]
444 filename = self.filenameList[0]
444 wait += 1
445 wait += 1
445
446
446 self.__filename_online = filename
447 self.__filename_online = filename
447
448
448 self.amisrFilePointer = h5py.File(filename,'r')
449 self.amisrFilePointer = h5py.File(filename,'r')
449 self.flagIsNewFile = 1
450 self.flagIsNewFile = 1
450 self.filename = filename
451 self.filename = filename
451 print("Setting the file: %s"%self.filename)
452 print("Setting the file: %s"%self.filename)
452 return 1
453 return 1
453
454
454
455
455 def readData(self):
456 def readData(self):
456 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
457 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
457 re = buffer[:,:,:,0]
458 re = buffer[:,:,:,0]
458 im = buffer[:,:,:,1]
459 im = buffer[:,:,:,1]
459 dataset = re + im*1j
460 dataset = re + im*1j
460
461
461 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
462 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
462 timeset = self.radacTime[:,0]
463 timeset = self.radacTime[:,0]
463
464
464 return dataset,timeset
465 return dataset,timeset
465
466
466 def reshapeData(self):
467 def reshapeData(self):
467 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
468 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
468 channels = self.beamCodeByPulse[0,:]
469 channels = self.beamCodeByPulse[0,:]
469 nchan = self.nchannels
470 nchan = self.nchannels
470 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
471 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
471 nblocks = self.nblocks
472 nblocks = self.nblocks
472 nsamples = self.nsa
473 nsamples = self.nsa
473
474
474 #Dimensions : nChannels, nProfiles, nSamples
475 #Dimensions : nChannels, nProfiles, nSamples
475 new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
476 new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
476 ############################################
477 ############################################
477
478
478 for thisChannel in range(nchan):
479 for thisChannel in range(nchan):
479 new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
480 new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
480
481
481
482
482 new_block = numpy.transpose(new_block, (1,0,2,3))
483 new_block = numpy.transpose(new_block, (1,0,2,3))
483 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
484 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
484
485
485 return new_block
486 return new_block
486
487
487 def updateIndexes(self):
488 def updateIndexes(self):
488
489
489 pass
490 pass
490
491
491 def fillJROHeader(self):
492 def fillJROHeader(self):
492
493
493 #fill radar controller header
494 #fill radar controller header
494 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
495 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
495 txA=self.__txA,
496 txA=self.__txA,
496 txB=0,
497 txB=0,
497 nWindows=1,
498 nWindows=1,
498 nHeights=self.__nSamples,
499 nHeights=self.__nSamples,
499 firstHeight=self.__firstHeight,
500 firstHeight=self.__firstHeight,
500 deltaHeight=self.__deltaHeight,
501 deltaHeight=self.__deltaHeight,
501 codeType=self.__codeType,
502 codeType=self.__codeType,
502 nCode=self.__nCode, nBaud=self.__nBaud,
503 nCode=self.__nCode, nBaud=self.__nBaud,
503 code = self.__code,
504 code = self.__code,
504 fClock=1)
505 fClock=1)
505
506 #fill system header
506 #fill system header
507 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
507 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
508 nProfiles=self.newProfiles,
508 nProfiles=self.newProfiles,
509 nChannels=len(self.__channelList),
509 nChannels=len(self.__channelList),
510 adcResolution=14,
510 adcResolution=14,
511 pciDioBusWidth=32)
511 pciDioBusWidth=32)
512
512
513 self.dataOut.type = "Voltage"
513 self.dataOut.type = "Voltage"
514 self.dataOut.data = None
514 self.dataOut.data = None
515 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
515 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
516 # self.dataOut.nChannels = 0
516 # self.dataOut.nChannels = 0
517
517
518 # self.dataOut.nHeights = 0
518 # self.dataOut.nHeights = 0
519
519
520 self.dataOut.nProfiles = self.newProfiles*self.nblocks
520 self.dataOut.nProfiles = self.newProfiles*self.nblocks
521 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
521 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
522 ranges = numpy.reshape(self.rangeFromFile[()],(-1))
522 ranges = numpy.reshape(self.rangeFromFile[()],(-1))
523 self.dataOut.heightList = ranges/1000.0 #km
523 self.dataOut.heightList = ranges/1000.0 #km
524 self.dataOut.channelList = self.__channelList
524 self.dataOut.channelList = self.__channelList
525 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
525 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
526
526
527 # self.dataOut.channelIndexList = None
527 # self.dataOut.channelIndexList = None
528
528
529
529
530 self.dataOut.azimuthList = numpy.array(self.azimuthList)
530 self.dataOut.azimuthList = numpy.array(self.azimuthList)
531 self.dataOut.elevationList = numpy.array(self.elevationList)
531 self.dataOut.elevationList = numpy.array(self.elevationList)
532 self.dataOut.codeList = numpy.array(self.beamCode)
532 self.dataOut.codeList = numpy.array(self.beamCode)
533 #print(self.dataOut.elevationList)
533 #print(self.dataOut.elevationList)
534 self.dataOut.flagNoData = True
534 self.dataOut.flagNoData = True
535
535
536 #Set to TRUE if the data is discontinuous
536 #Set to TRUE if the data is discontinuous
537 self.dataOut.flagDiscontinuousBlock = False
537 self.dataOut.flagDiscontinuousBlock = False
538
538
539 self.dataOut.utctime = None
539 self.dataOut.utctime = None
540
540
541 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
541 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
542 if self.timezone == 'lt':
542 if self.timezone == 'lt':
543 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
543 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
544 else:
544 else:
545 self.dataOut.timeZone = 0 #by default time is UTC
545 self.dataOut.timeZone = 0 #by default time is UTC
546
546
547 self.dataOut.dstFlag = 0
547 self.dataOut.dstFlag = 0
548 self.dataOut.errorCount = 0
548 self.dataOut.errorCount = 0
549 self.dataOut.nCohInt = 1
549 self.dataOut.nCohInt = 1
550 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
550 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
551 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
551 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
552 self.dataOut.flagShiftFFT = False
552 self.dataOut.flagShiftFFT = False
553 self.dataOut.ippSeconds = self.ippSeconds
553 self.dataOut.ippSeconds = self.ippSeconds
554
554
555 #Time interval between profiles
555 #Time interval between profiles
556 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
556 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
557
557
558 self.dataOut.frequency = self.__frequency
558 self.dataOut.frequency = self.__frequency
559 self.dataOut.realtime = self.online
559 self.dataOut.realtime = self.online
560 pass
560 pass
561
561
562 def readNextFile(self,online=False):
562 def readNextFile(self,online=False):
563
563
564 if not(online):
564 if not(online):
565 newFile = self.__setNextFileOffline()
565 newFile = self.__setNextFileOffline()
566 else:
566 else:
567 newFile = self.__setNextFileOnline()
567 newFile = self.__setNextFileOnline()
568
568
569 if not(newFile):
569 if not(newFile):
570 self.dataOut.error = True
570 self.dataOut.error = True
571 return 0
571 return 0
572
572
573 if not self.readAMISRHeader(self.amisrFilePointer):
573 if not self.readAMISRHeader(self.amisrFilePointer):
574 self.dataOut.error = True
574 self.dataOut.error = True
575 return 0
575 return 0
576
576
577 self.createBuffers()
577 self.createBuffers()
578 self.fillJROHeader()
578 self.fillJROHeader()
579
579
580 #self.__firstFile = False
580 #self.__firstFile = False
581
581
582
582
583
583
584 self.dataset,self.timeset = self.readData()
584 self.dataset,self.timeset = self.readData()
585
585
586 if self.endDate!=None:
586 if self.endDate!=None:
587 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
587 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
588 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
588 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
589 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
589 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
590 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
590 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
591 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
591 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
592 if self.timezone == 'lt':
592 if self.timezone == 'lt':
593 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
593 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
594 if (startDateTime_File>endDateTime_Reader):
594 if (startDateTime_File>endDateTime_Reader):
595 return 0
595 return 0
596
596
597 self.jrodataset = self.reshapeData()
597 self.jrodataset = self.reshapeData()
598 #----self.updateIndexes()
598 #----self.updateIndexes()
599 self.profileIndex = 0
599 self.profileIndex = 0
600
600
601 return 1
601 return 1
602
602
603
603
604 def __hasNotDataInBuffer(self):
604 def __hasNotDataInBuffer(self):
605 if self.profileIndex >= (self.newProfiles*self.nblocks):
605 if self.profileIndex >= (self.newProfiles*self.nblocks):
606 return 1
606 return 1
607 return 0
607 return 0
608
608
609
609
610 def getData(self):
610 def getData(self):
611
611
612 if self.flagNoMoreFiles:
612 if self.flagNoMoreFiles:
613 self.dataOut.flagNoData = True
613 self.dataOut.flagNoData = True
614 return 0
614 return 0
615
615
616 if self.__hasNotDataInBuffer():
616 if self.__hasNotDataInBuffer():
617 if not (self.readNextFile(self.online)):
617 if not (self.readNextFile(self.online)):
618 return 0
618 return 0
619
619
620
620
621 if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
621 if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
622 self.dataOut.flagNoData = True
622 self.dataOut.flagNoData = True
623 return 0
623 return 0
624
624
625 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
625 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
626
626
627 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
627 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
628
628
629 #print("R_t",self.timeset)
629 #print("R_t",self.timeset)
630
630
631 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
631 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
632 #verificar basic header de jro data y ver si es compatible con este valor
632 #verificar basic header de jro data y ver si es compatible con este valor
633 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
633 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
634 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
634 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
635 indexblock = self.profileIndex/self.newProfiles
635 indexblock = self.profileIndex/self.newProfiles
636 #print (indexblock, indexprof)
636 #print (indexblock, indexprof)
637 diffUTC = 0
637 diffUTC = 0
638 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
638 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
639
639
640 #print("utc :",indexblock," __ ",t_comp)
640 #print("utc :",indexblock," __ ",t_comp)
641 #print(numpy.shape(self.timeset))
641 #print(numpy.shape(self.timeset))
642 self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
642 self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
643 #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
643 #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
644
644
645 self.dataOut.profileIndex = self.profileIndex
645 self.dataOut.profileIndex = self.profileIndex
646 #print("N profile:",self.profileIndex,self.newProfiles,self.nblocks,self.dataOut.utctime)
646 #print("N profile:",self.profileIndex,self.newProfiles,self.nblocks,self.dataOut.utctime)
647 self.dataOut.flagNoData = False
647 self.dataOut.flagNoData = False
648 # if indexprof == 0:
648 # if indexprof == 0:
649 # print("kamisr: ",self.dataOut.utctime)
649 # print("kamisr: ",self.dataOut.utctime)
650
650
651 self.profileIndex += 1
651 self.profileIndex += 1
652
652
653 return self.dataOut.data #retorno necesario??
653 return self.dataOut.data #retorno necesario??
654
654
655
655
656 def run(self, **kwargs):
656 def run(self, **kwargs):
657 '''
657 '''
658 This method will be called many times so here you should put all your code
658 This method will be called many times so here you should put all your code
659 '''
659 '''
660 #print("running kamisr")
660 #print("running kamisr")
661 if not self.isConfig:
661 if not self.isConfig:
662 self.setup(**kwargs)
662 self.setup(**kwargs)
663 self.isConfig = True
663 self.isConfig = True
664
664
665 self.getData()
665 self.getData()
Show file before | Show file after | Hide comments
@@ -1,685 +1,685
1 import os
1 import os
2 import time
2 import time
3 import datetime
3 import datetime
4
4
5 import numpy
5 import numpy
6 import h5py
6 import h5py
7
7
8 import schainpy.admin
8 import schainpy.admin
9 from schainpy.model.data.jrodata import *
9 from schainpy.model.data.jrodata import *
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 from schainpy.utils import log
12 from schainpy.utils import log
13
13
14
14
15 class HDFReader(Reader, ProcessingUnit):
15 class HDFReader(Reader, ProcessingUnit):
16 """Processing unit to read HDF5 format files
16 """Processing unit to read HDF5 format files
17
17
18 This unit reads HDF5 files created with `HDFWriter` operation contains
18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 attributes.
20 attributes.
21 It is possible to read any HDF5 file by given the structure in the `description`
21 It is possible to read any HDF5 file by given the structure in the `description`
22 parameter, also you can add extra values to metadata with the parameter `extras`.
22 parameter, also you can add extra values to metadata with the parameter `extras`.
23
23
24 Parameters:
24 Parameters:
25 -----------
25 -----------
26 path : str
26 path : str
27 Path where files are located.
27 Path where files are located.
28 startDate : date
28 startDate : date
29 Start date of the files
29 Start date of the files
30 endDate : list
30 endDate : list
31 End date of the files
31 End date of the files
32 startTime : time
32 startTime : time
33 Start time of the files
33 Start time of the files
34 endTime : time
34 endTime : time
35 End time of the files
35 End time of the files
36 description : dict, optional
36 description : dict, optional
37 Dictionary with the description of the HDF5 file
37 Dictionary with the description of the HDF5 file
38 extras : dict, optional
38 extras : dict, optional
39 Dictionary with extra metadata to be be added to `dataOut`
39 Dictionary with extra metadata to be be added to `dataOut`
40
40
41 Examples
41 Examples
42 --------
42 --------
43
43
44 desc = {
44 desc = {
45 'Data': {
45 'Data': {
46 'data_output': ['u', 'v', 'w'],
46 'data_output': ['u', 'v', 'w'],
47 'utctime': 'timestamps',
47 'utctime': 'timestamps',
48 } ,
48 } ,
49 'Metadata': {
49 'Metadata': {
50 'heightList': 'heights'
50 'heightList': 'heights'
51 }
51 }
52 }
52 }
53
53
54 desc = {
54 desc = {
55 'Data': {
55 'Data': {
56 'data_output': 'winds',
56 'data_output': 'winds',
57 'utctime': 'timestamps'
57 'utctime': 'timestamps'
58 },
58 },
59 'Metadata': {
59 'Metadata': {
60 'heightList': 'heights'
60 'heightList': 'heights'
61 }
61 }
62 }
62 }
63
63
64 extras = {
64 extras = {
65 'timeZone': 300
65 'timeZone': 300
66 }
66 }
67
67
68 reader = project.addReadUnit(
68 reader = project.addReadUnit(
69 name='HDFReader',
69 name='HDFReader',
70 path='/path/to/files',
70 path='/path/to/files',
71 startDate='2019/01/01',
71 startDate='2019/01/01',
72 endDate='2019/01/31',
72 endDate='2019/01/31',
73 startTime='00:00:00',
73 startTime='00:00:00',
74 endTime='23:59:59',
74 endTime='23:59:59',
75 # description=json.dumps(desc),
75 # description=json.dumps(desc),
76 # extras=json.dumps(extras),
76 # extras=json.dumps(extras),
77 )
77 )
78
78
79 """
79 """
80
80
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82
82
83 def __init__(self):
83 def __init__(self):
84 ProcessingUnit.__init__(self)
84 ProcessingUnit.__init__(self)
85
85
86 self.ext = ".hdf5"
86 self.ext = ".hdf5"
87 self.optchar = "D"
87 self.optchar = "D"
88 self.meta = {}
88 self.meta = {}
89 self.data = {}
89 self.data = {}
90 self.open_file = h5py.File
90 self.open_file = h5py.File
91 self.open_mode = 'r'
91 self.open_mode = 'r'
92 self.description = {}
92 self.description = {}
93 self.extras = {}
93 self.extras = {}
94 self.filefmt = "*%Y%j***"
94 self.filefmt = "*%Y%j***"
95 self.folderfmt = "*%Y%j"
95 self.folderfmt = "*%Y%j"
96 self.utcoffset = 0
96 self.utcoffset = 0
97
97
98 self.dataOut = Parameters()
98 self.dataOut = Parameters()
99 self.dataOut.error=False ## NOTE: Importante definir esto antes inicio
99 self.dataOut.error=False ## NOTE: Importante definir esto antes inicio
100 self.dataOut.flagNoData = True
100 self.dataOut.flagNoData = True
101
101
102 def setup(self, **kwargs):
102 def setup(self, **kwargs):
103
103
104 self.set_kwargs(**kwargs)
104 self.set_kwargs(**kwargs)
105 if not self.ext.startswith('.'):
105 if not self.ext.startswith('.'):
106 self.ext = '.{}'.format(self.ext)
106 self.ext = '.{}'.format(self.ext)
107
107
108 if self.online:
108 if self.online:
109 log.log("Searching files in online mode...", self.name)
109 log.log("Searching files in online mode...", self.name)
110
110
111 for nTries in range(self.nTries):
111 for nTries in range(self.nTries):
112 fullpath = self.searchFilesOnLine(self.path, self.startDate,
112 fullpath = self.searchFilesOnLine(self.path, self.startDate,
113 self.endDate, self.expLabel, self.ext, self.walk,
113 self.endDate, self.expLabel, self.ext, self.walk,
114 self.filefmt, self.folderfmt)
114 self.filefmt, self.folderfmt)
115 pathname, filename = os.path.split(fullpath)
115 pathname, filename = os.path.split(fullpath)
116
116
117 try:
117 try:
118 fullpath = next(fullpath)
118 fullpath = next(fullpath)
119
119
120 except:
120 except:
121 fullpath = None
121 fullpath = None
122
122
123 if fullpath:
123 if fullpath:
124 break
124 break
125
125
126 log.warning(
126 log.warning(
127 'Waiting {} sec for a valid file in {}: try {} ...'.format(
127 'Waiting {} sec for a valid file in {}: try {} ...'.format(
128 self.delay, self.path, nTries + 1),
128 self.delay, self.path, nTries + 1),
129 self.name)
129 self.name)
130 time.sleep(self.delay)
130 time.sleep(self.delay)
131
131
132 if not(fullpath):
132 if not(fullpath):
133 raise schainpy.admin.SchainError(
133 raise schainpy.admin.SchainError(
134 'There isn\'t any valid file in {}'.format(self.path))
134 'There isn\'t any valid file in {}'.format(self.path))
135
135
136 pathname, filename = os.path.split(fullpath)
136 pathname, filename = os.path.split(fullpath)
137 self.year = int(filename[1:5])
137 self.year = int(filename[1:5])
138 self.doy = int(filename[5:8])
138 self.doy = int(filename[5:8])
139 self.set = int(filename[8:11]) - 1
139 self.set = int(filename[8:11]) - 1
140 else:
140 else:
141 log.log("Searching files in {}".format(self.path), self.name)
141 log.log("Searching files in {}".format(self.path), self.name)
142 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
142 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
143 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
143 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
144
144
145 self.setNextFile()
145 self.setNextFile()
146
146
147
147
148
148
149
149
150 def readFirstHeader(self):
150 def readFirstHeader(self):
151 '''Read metadata and data'''
151 '''Read metadata and data'''
152
152
153 self.__readMetadata()
153 self.__readMetadata()
154 self.__readData()
154 self.__readData()
155 self.__setBlockList()
155 self.__setBlockList()
156
156
157 for attr in self.meta:
157 for attr in self.meta:
158 setattr(self.dataOut, attr, self.meta[attr])
158 setattr(self.dataOut, attr, self.meta[attr])
159 self.blockIndex = 0
159 self.blockIndex = 0
160
160
161 return
161 return
162
162
163 def __setBlockList(self):
163 def __setBlockList(self):
164 '''
164 '''
165 Selects the data within the times defined
165 Selects the data within the times defined
166
166
167 self.fp
167 self.fp
168 self.startTime
168 self.startTime
169 self.endTime
169 self.endTime
170 self.blockList
170 self.blockList
171 self.blocksPerFile
171 self.blocksPerFile
172
172
173 '''
173 '''
174
174
175 startTime = self.startTime
175 startTime = self.startTime
176 endTime = self.endTime
176 endTime = self.endTime
177 thisUtcTime = self.data['utctime'] + self.utcoffset
177 thisUtcTime = self.data['utctime'] + self.utcoffset
178 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
178 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
179 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
179 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
180 self.startFileDatetime = thisDatetime
180 self.startFileDatetime = thisDatetime
181 thisDate = thisDatetime.date()
181 thisDate = thisDatetime.date()
182 thisTime = thisDatetime.time()
182 thisTime = thisDatetime.time()
183
183
184 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
184 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
185 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
185 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
186
186
187 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
187 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
188
188
189 self.blockList = ind
189 self.blockList = ind
190 self.blocksPerFile = len(ind)
190 self.blocksPerFile = len(ind)
191 self.blocksPerFile = len(thisUtcTime)
191 self.blocksPerFile = len(thisUtcTime)
192 return
192 return
193
193
194 def __readMetadata(self):
194 def __readMetadata(self):
195 '''
195 '''
196 Reads Metadata
196 Reads Metadata
197 '''
197 '''
198
198
199 meta = {}
199 meta = {}
200
200
201 if self.description:
201 if self.description:
202 for key, value in self.description['Metadata'].items():
202 for key, value in self.description['Metadata'].items():
203 meta[key] = self.fp[value][()]
203 meta[key] = self.fp[value][()]
204 else:
204 else:
205 grp = self.fp['Metadata']
205 grp = self.fp['Metadata']
206 for name in grp:
206 for name in grp:
207 meta[name] = grp[name][()]
207 meta[name] = grp[name][()]
208
208
209 if self.extras:
209 if self.extras:
210 for key, value in self.extras.items():
210 for key, value in self.extras.items():
211 meta[key] = value
211 meta[key] = value
212 self.meta = meta
212 self.meta = meta
213
213
214 return
214 return
215
215
216
216
217
217
218 def checkForRealPath(self, nextFile, nextDay):
218 def checkForRealPath(self, nextFile, nextDay):
219
219
220 # print("check FRP")
220 # print("check FRP")
221 # dt = self.startFileDatetime + datetime.timedelta(1)
221 # dt = self.startFileDatetime + datetime.timedelta(1)
222 # filename = '{}.{}{}'.format(self.path, dt.strftime('%Y%m%d'), self.ext)
222 # filename = '{}.{}{}'.format(self.path, dt.strftime('%Y%m%d'), self.ext)
223 # fullfilename = os.path.join(self.path, filename)
223 # fullfilename = os.path.join(self.path, filename)
224 # print("check Path ",fullfilename,filename)
224 # print("check Path ",fullfilename,filename)
225 # if os.path.exists(fullfilename):
225 # if os.path.exists(fullfilename):
226 # return fullfilename, filename
226 # return fullfilename, filename
227 # return None, filename
227 # return None, filename
228 return None,None
228 return None,None
229
229
230 def __readData(self):
230 def __readData(self):
231
231
232 data = {}
232 data = {}
233
233
234 if self.description:
234 if self.description:
235 for key, value in self.description['Data'].items():
235 for key, value in self.description['Data'].items():
236 if isinstance(value, str):
236 if isinstance(value, str):
237 if isinstance(self.fp[value], h5py.Dataset):
237 if isinstance(self.fp[value], h5py.Dataset):
238 data[key] = self.fp[value][()]
238 data[key] = self.fp[value][()]
239 elif isinstance(self.fp[value], h5py.Group):
239 elif isinstance(self.fp[value], h5py.Group):
240 array = []
240 array = []
241 for ch in self.fp[value]:
241 for ch in self.fp[value]:
242 array.append(self.fp[value][ch][()])
242 array.append(self.fp[value][ch][()])
243 data[key] = numpy.array(array)
243 data[key] = numpy.array(array)
244 elif isinstance(value, list):
244 elif isinstance(value, list):
245 array = []
245 array = []
246 for ch in value:
246 for ch in value:
247 array.append(self.fp[ch][()])
247 array.append(self.fp[ch][()])
248 data[key] = numpy.array(array)
248 data[key] = numpy.array(array)
249 else:
249 else:
250 grp = self.fp['Data']
250 grp = self.fp['Data']
251 for name in grp:
251 for name in grp:
252 if isinstance(grp[name], h5py.Dataset):
252 if isinstance(grp[name], h5py.Dataset):
253 array = grp[name][()]
253 array = grp[name][()]
254 elif isinstance(grp[name], h5py.Group):
254 elif isinstance(grp[name], h5py.Group):
255 array = []
255 array = []
256 for ch in grp[name]:
256 for ch in grp[name]:
257 array.append(grp[name][ch][()])
257 array.append(grp[name][ch][()])
258 array = numpy.array(array)
258 array = numpy.array(array)
259 else:
259 else:
260 log.warning('Unknown type: {}'.format(name))
260 log.warning('Unknown type: {}'.format(name))
261
261
262 if name in self.description:
262 if name in self.description:
263 key = self.description[name]
263 key = self.description[name]
264 else:
264 else:
265 key = name
265 key = name
266 data[key] = array
266 data[key] = array
267
267
268 self.data = data
268 self.data = data
269 return
269 return
270
270
271 def getData(self):
271 def getData(self):
272 if not self.isDateTimeInRange(self.startFileDatetime, self.startDate, self.endDate, self.startTime, self.endTime):
272 if not self.isDateTimeInRange(self.startFileDatetime, self.startDate, self.endDate, self.startTime, self.endTime):
273 self.dataOut.flagNoData = True
273 self.dataOut.flagNoData = True
274 self.blockIndex = self.blocksPerFile
274 self.blockIndex = self.blocksPerFile
275 self.dataOut.error = True # TERMINA EL PROGRAMA
275 self.dataOut.error = True # TERMINA EL PROGRAMA
276 return
276 return
277 for attr in self.data:
277 for attr in self.data:
278
278
279 if self.data[attr].ndim == 1:
279 if self.data[attr].ndim == 1:
280 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
280 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
281 else:
281 else:
282 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
282 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
283
283
284
284
285 self.blockIndex += 1
285 self.blockIndex += 1
286
286
287 if self.blockIndex == 1:
287 if self.blockIndex == 1:
288 log.log("Block No. {}/{} -> {}".format(
288 log.log("Block No. {}/{} -> {}".format(
289 self.blockIndex,
289 self.blockIndex,
290 self.blocksPerFile,
290 self.blocksPerFile,
291 self.dataOut.datatime.ctime()), self.name)
291 self.dataOut.datatime.ctime()), self.name)
292 else:
292 else:
293 log.log("Block No. {}/{} ".format(
293 log.log("Block No. {}/{} ".format(
294 self.blockIndex,
294 self.blockIndex,
295 self.blocksPerFile),self.name)
295 self.blocksPerFile),self.name)
296
296
297 if self.blockIndex == self.blocksPerFile:
297 if self.blockIndex == self.blocksPerFile:
298 self.setNextFile()
298 self.setNextFile()
299
299
300 self.dataOut.flagNoData = False
300 self.dataOut.flagNoData = False
301
301
302
302
303 def run(self, **kwargs):
303 def run(self, **kwargs):
304
304
305 if not(self.isConfig):
305 if not(self.isConfig):
306 self.setup(**kwargs)
306 self.setup(**kwargs)
307 self.isConfig = True
307 self.isConfig = True
308
308
309 self.getData()
309 self.getData()
310
310
311 #@MPDecorator
311 #@MPDecorator
312 class HDFWrite(Operation):
312 class HDFWrite(Operation):
313 """Operation to write HDF5 files.
313 """Operation to write HDF5 files.
314
314
315 The HDF5 file contains by default two groups Data and Metadata where
315 The HDF5 file contains by default two groups Data and Metadata where
316 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
316 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
317 parameters, data attributes are normaly time dependent where the metadata
317 parameters, data attributes are normaly time dependent where the metadata
318 are not.
318 are not.
319 It is possible to customize the structure of the HDF5 file with the
319 It is possible to customize the structure of the HDF5 file with the
320 optional description parameter see the examples.
320 optional description parameter see the examples.
321
321
322 Parameters:
322 Parameters:
323 -----------
323 -----------
324 path : str
324 path : str
325 Path where files will be saved.
325 Path where files will be saved.
326 blocksPerFile : int
326 blocksPerFile : int
327 Number of blocks per file
327 Number of blocks per file
328 metadataList : list
328 metadataList : list
329 List of the dataOut attributes that will be saved as metadata
329 List of the dataOut attributes that will be saved as metadata
330 dataList : int
330 dataList : int
331 List of the dataOut attributes that will be saved as data
331 List of the dataOut attributes that will be saved as data
332 setType : bool
332 setType : bool
333 If True the name of the files corresponds to the timestamp of the data
333 If True the name of the files corresponds to the timestamp of the data
334 description : dict, optional
334 description : dict, optional
335 Dictionary with the desired description of the HDF5 file
335 Dictionary with the desired description of the HDF5 file
336
336
337 Examples
337 Examples
338 --------
338 --------
339
339
340 desc = {
340 desc = {
341 'data_output': {'winds': ['z', 'w', 'v']},
341 'data_output': {'winds': ['z', 'w', 'v']},
342 'utctime': 'timestamps',
342 'utctime': 'timestamps',
343 'heightList': 'heights'
343 'heightList': 'heights'
344 }
344 }
345 desc = {
345 desc = {
346 'data_output': ['z', 'w', 'v'],
346 'data_output': ['z', 'w', 'v'],
347 'utctime': 'timestamps',
347 'utctime': 'timestamps',
348 'heightList': 'heights'
348 'heightList': 'heights'
349 }
349 }
350 desc = {
350 desc = {
351 'Data': {
351 'Data': {
352 'data_output': 'winds',
352 'data_output': 'winds',
353 'utctime': 'timestamps'
353 'utctime': 'timestamps'
354 },
354 },
355 'Metadata': {
355 'Metadata': {
356 'heightList': 'heights'
356 'heightList': 'heights'
357 }
357 }
358 }
358 }
359
359
360 writer = proc_unit.addOperation(name='HDFWriter')
360 writer = proc_unit.addOperation(name='HDFWriter')
361 writer.addParameter(name='path', value='/path/to/file')
361 writer.addParameter(name='path', value='/path/to/file')
362 writer.addParameter(name='blocksPerFile', value='32')
362 writer.addParameter(name='blocksPerFile', value='32')
363 writer.addParameter(name='metadataList', value='heightList,timeZone')
363 writer.addParameter(name='metadataList', value='heightList,timeZone')
364 writer.addParameter(name='dataList',value='data_output,utctime')
364 writer.addParameter(name='dataList',value='data_output,utctime')
365 # writer.addParameter(name='description',value=json.dumps(desc))
365 # writer.addParameter(name='description',value=json.dumps(desc))
366
366
367 """
367 """
368
368
369 ext = ".hdf5"
369 ext = ".hdf5"
370 optchar = "D"
370 optchar = "D"
371 filename = None
371 filename = None
372 path = None
372 path = None
373 setFile = None
373 setFile = None
374 fp = None
374 fp = None
375 firsttime = True
375 firsttime = True
376 #Configurations
376 #Configurations
377 blocksPerFile = None
377 blocksPerFile = None
378 blockIndex = None
378 blockIndex = None
379 dataOut = None #eval ??????
379 dataOut = None #eval ??????
380 #Data Arrays
380 #Data Arrays
381 dataList = None
381 dataList = None
382 metadataList = None
382 metadataList = None
383 currentDay = None
383 currentDay = None
384 lastTime = None
384 lastTime = None
385 timeZone = "ut"
385 timeZone = "ut"
386 hourLimit = 3
386 hourLimit = 3
387 breakDays = True
387 breakDays = True
388
388
389 def __init__(self):
389 def __init__(self):
390
390
391 Operation.__init__(self)
391 Operation.__init__(self)
392
392
393
393
394 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None,
394 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None,
395 description={},timeZone = "ut",hourLimit = 3, breakDays=True):
395 description={},timeZone = "ut",hourLimit = 3, breakDays=True):
396 self.path = path
396 self.path = path
397 self.blocksPerFile = blocksPerFile
397 self.blocksPerFile = blocksPerFile
398 self.metadataList = metadataList
398 self.metadataList = metadataList
399 self.dataList = [s.strip() for s in dataList]
399 self.dataList = [s.strip() for s in dataList]
400 self.setType = setType
400 self.setType = setType
401 self.description = description
401 self.description = description
402 self.timeZone = timeZone
402 self.timeZone = timeZone
403 self.hourLimit = hourLimit
403 self.hourLimit = hourLimit
404 self.breakDays = breakDays
404 self.breakDays = breakDays
405
405
406 if self.metadataList is None:
406 if self.metadataList is None:
407 self.metadataList = self.dataOut.metadata_list
407 self.metadataList = self.dataOut.metadata_list
408
408
409 tableList = []
409 tableList = []
410 dsList = []
410 dsList = []
411
411
412 for i in range(len(self.dataList)):
412 for i in range(len(self.dataList)):
413 dsDict = {}
413 dsDict = {}
414 if hasattr(self.dataOut, self.dataList[i]):
414 if hasattr(self.dataOut, self.dataList[i]):
415 dataAux = getattr(self.dataOut, self.dataList[i])
415 dataAux = getattr(self.dataOut, self.dataList[i])
416 dsDict['variable'] = self.dataList[i]
416 dsDict['variable'] = self.dataList[i]
417 else:
417 else:
418 log.warning('Attribute {} not found in dataOut', self.name)
418 log.warning('Attribute {} not found in dataOut'.format(self.dataList[i]),self.name)
419 continue
419 continue
420
420
421 if dataAux is None:
421 if dataAux is None:
422 continue
422 continue
423 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
423 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
424 dsDict['nDim'] = 0
424 dsDict['nDim'] = 0
425 else:
425 else:
426 dsDict['nDim'] = len(dataAux.shape)
426 dsDict['nDim'] = len(dataAux.shape)
427 dsDict['shape'] = dataAux.shape
427 dsDict['shape'] = dataAux.shape
428 dsDict['dsNumber'] = dataAux.shape[0]
428 dsDict['dsNumber'] = dataAux.shape[0]
429 dsDict['dtype'] = dataAux.dtype
429 dsDict['dtype'] = dataAux.dtype
430
430
431 dsList.append(dsDict)
431 dsList.append(dsDict)
432
432
433 self.blockIndex = 0
433 self.blockIndex = 0
434 self.dsList = dsList
434 self.dsList = dsList
435 self.currentDay = self.dataOut.datatime.date()
435 self.currentDay = self.dataOut.datatime.date()
436
436
437
437
438 def timeFlag(self):
438 def timeFlag(self):
439 currentTime = self.dataOut.utctime
439 currentTime = self.dataOut.utctime
440 timeTuple = None
440 timeTuple = None
441 if self.timeZone == "lt":
441 if self.timeZone == "lt":
442 timeTuple = time.localtime(currentTime)
442 timeTuple = time.localtime(currentTime)
443 else :
443 else :
444 timeTuple = time.gmtime(currentTime)
444 timeTuple = time.gmtime(currentTime)
445
445
446 dataDay = timeTuple.tm_yday
446 dataDay = timeTuple.tm_yday
447
447
448 if self.lastTime is None:
448 if self.lastTime is None:
449 self.lastTime = currentTime
449 self.lastTime = currentTime
450 self.currentDay = dataDay
450 self.currentDay = dataDay
451 return False
451 return False
452
452
453 timeDiff = currentTime - self.lastTime
453 timeDiff = currentTime - self.lastTime
454
454
455 #Si el dia es diferente o si la diferencia entre un dato y otro supera self.hourLimit
455 #Si el dia es diferente o si la diferencia entre un dato y otro supera self.hourLimit
456 if (dataDay != self.currentDay) and self.breakDays:
456 if (dataDay != self.currentDay) and self.breakDays:
457 self.currentDay = dataDay
457 self.currentDay = dataDay
458 return True
458 return True
459 elif timeDiff > self.hourLimit*60*60:
459 elif timeDiff > self.hourLimit*60*60:
460 self.lastTime = currentTime
460 self.lastTime = currentTime
461 return True
461 return True
462 else:
462 else:
463 self.lastTime = currentTime
463 self.lastTime = currentTime
464 return False
464 return False
465
465
466 def run(self, dataOut,**kwargs):
466 def run(self, dataOut,**kwargs):
467
467
468 self.dataOut = dataOut
468 self.dataOut = dataOut
469 if not(self.isConfig):
469 if not(self.isConfig):
470 self.setup(**kwargs)
470 self.setup(**kwargs)
471
471
472 self.isConfig = True
472 self.isConfig = True
473 self.setNextFile()
473 self.setNextFile()
474
474
475 self.putData()
475 self.putData()
476
476
477 return self.dataOut
477 return self.dataOut
478
478
479 def setNextFile(self):
479 def setNextFile(self):
480
480
481 ext = self.ext
481 ext = self.ext
482 path = self.path
482 path = self.path
483 setFile = self.setFile
483 setFile = self.setFile
484 timeTuple = None
484 timeTuple = None
485 if self.timeZone == "lt":
485 if self.timeZone == "lt":
486 timeTuple = time.localtime(self.dataOut.utctime)
486 timeTuple = time.localtime(self.dataOut.utctime)
487 elif self.timeZone == "ut":
487 elif self.timeZone == "ut":
488 timeTuple = time.gmtime(self.dataOut.utctime)
488 timeTuple = time.gmtime(self.dataOut.utctime)
489 #print("path: ",timeTuple)
489 #print("path: ",timeTuple)
490 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
490 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
491 fullpath = os.path.join(path, subfolder)
491 fullpath = os.path.join(path, subfolder)
492
492
493 if os.path.exists(fullpath):
493 if os.path.exists(fullpath):
494 filesList = os.listdir(fullpath)
494 filesList = os.listdir(fullpath)
495 filesList = [k for k in filesList if k.startswith(self.optchar)]
495 filesList = [k for k in filesList if k.startswith(self.optchar)]
496 if len( filesList ) > 0:
496 if len( filesList ) > 0:
497 filesList = sorted(filesList, key=str.lower)
497 filesList = sorted(filesList, key=str.lower)
498 filen = filesList[-1]
498 filen = filesList[-1]
499 # el filename debera tener el siguiente formato
499 # el filename debera tener el siguiente formato
500 # 0 1234 567 89A BCDE (hex)
500 # 0 1234 567 89A BCDE (hex)
501 # x YYYY DDD SSS .ext
501 # x YYYY DDD SSS .ext
502 if isNumber(filen[8:11]):
502 if isNumber(filen[8:11]):
503 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
503 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
504 else:
504 else:
505 setFile = -1
505 setFile = -1
506 else:
506 else:
507 setFile = -1 #inicializo mi contador de seteo
507 setFile = -1 #inicializo mi contador de seteo
508 else:
508 else:
509 os.makedirs(fullpath)
509 os.makedirs(fullpath)
510 setFile = -1 #inicializo mi contador de seteo
510 setFile = -1 #inicializo mi contador de seteo
511
511
512 if self.setType is None:
512 if self.setType is None:
513 setFile += 1
513 setFile += 1
514 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
514 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
515 timeTuple.tm_year,
515 timeTuple.tm_year,
516 timeTuple.tm_yday,
516 timeTuple.tm_yday,
517 setFile,
517 setFile,
518 ext )
518 ext )
519 else:
519 else:
520 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
520 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
521 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
521 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
522 timeTuple.tm_year,
522 timeTuple.tm_year,
523 timeTuple.tm_yday,
523 timeTuple.tm_yday,
524 setFile,
524 setFile,
525 ext )
525 ext )
526
526
527 self.filename = os.path.join( path, subfolder, file )
527 self.filename = os.path.join( path, subfolder, file )
528
528
529
529
530
530
531 def getLabel(self, name, x=None):
531 def getLabel(self, name, x=None):
532
532
533 if x is None:
533 if x is None:
534 if 'Data' in self.description:
534 if 'Data' in self.description:
535 data = self.description['Data']
535 data = self.description['Data']
536 if 'Metadata' in self.description:
536 if 'Metadata' in self.description:
537 data.update(self.description['Metadata'])
537 data.update(self.description['Metadata'])
538 else:
538 else:
539 data = self.description
539 data = self.description
540 if name in data:
540 if name in data:
541 if isinstance(data[name], str):
541 if isinstance(data[name], str):
542 return data[name]
542 return data[name]
543 elif isinstance(data[name], list):
543 elif isinstance(data[name], list):
544 return None
544 return None
545 elif isinstance(data[name], dict):
545 elif isinstance(data[name], dict):
546 for key, value in data[name].items():
546 for key, value in data[name].items():
547 return key
547 return key
548 return name
548 return name
549 else:
549 else:
550 if 'Metadata' in self.description:
550 if 'Metadata' in self.description:
551 meta = self.description['Metadata']
551 meta = self.description['Metadata']
552 else:
552 else:
553 meta = self.description
553 meta = self.description
554 if name in meta:
554 if name in meta:
555 if isinstance(meta[name], list):
555 if isinstance(meta[name], list):
556 return meta[name][x]
556 return meta[name][x]
557 elif isinstance(meta[name], dict):
557 elif isinstance(meta[name], dict):
558 for key, value in meta[name].items():
558 for key, value in meta[name].items():
559 return value[x]
559 return value[x]
560 if 'cspc' in name:
560 if 'cspc' in name:
561 return 'pair{:02d}'.format(x)
561 return 'pair{:02d}'.format(x)
562 else:
562 else:
563 return 'channel{:02d}'.format(x)
563 return 'channel{:02d}'.format(x)
564
564
565 def writeMetadata(self, fp):
565 def writeMetadata(self, fp):
566
566
567 if self.description:
567 if self.description:
568 if 'Metadata' in self.description:
568 if 'Metadata' in self.description:
569 grp = fp.create_group('Metadata')
569 grp = fp.create_group('Metadata')
570 else:
570 else:
571 grp = fp
571 grp = fp
572 else:
572 else:
573 grp = fp.create_group('Metadata')
573 grp = fp.create_group('Metadata')
574
574
575 for i in range(len(self.metadataList)):
575 for i in range(len(self.metadataList)):
576 if not hasattr(self.dataOut, self.metadataList[i]):
576 if not hasattr(self.dataOut, self.metadataList[i]):
577 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
577 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
578 continue
578 continue
579 value = getattr(self.dataOut, self.metadataList[i])
579 value = getattr(self.dataOut, self.metadataList[i])
580 if isinstance(value, bool):
580 if isinstance(value, bool):
581 if value is True:
581 if value is True:
582 value = 1
582 value = 1
583 else:
583 else:
584 value = 0
584 value = 0
585 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
585 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
586 return
586 return
587
587
588 def writeData(self, fp):
588 def writeData(self, fp):
589
589
590 if self.description:
590 if self.description:
591 if 'Data' in self.description:
591 if 'Data' in self.description:
592 grp = fp.create_group('Data')
592 grp = fp.create_group('Data')
593 else:
593 else:
594 grp = fp
594 grp = fp
595 else:
595 else:
596 grp = fp.create_group('Data')
596 grp = fp.create_group('Data')
597
597
598 dtsets = []
598 dtsets = []
599 data = []
599 data = []
600
600
601 for dsInfo in self.dsList:
601 for dsInfo in self.dsList:
602 if dsInfo['nDim'] == 0:
602 if dsInfo['nDim'] == 0:
603 ds = grp.create_dataset(
603 ds = grp.create_dataset(
604 self.getLabel(dsInfo['variable']),
604 self.getLabel(dsInfo['variable']),
605 (self.blocksPerFile, ),
605 (self.blocksPerFile, ),
606 chunks=True,
606 chunks=True,
607 dtype=numpy.float64)
607 dtype=numpy.float64)
608 dtsets.append(ds)
608 dtsets.append(ds)
609 data.append((dsInfo['variable'], -1))
609 data.append((dsInfo['variable'], -1))
610 else:
610 else:
611 label = self.getLabel(dsInfo['variable'])
611 label = self.getLabel(dsInfo['variable'])
612 if label is not None:
612 if label is not None:
613 sgrp = grp.create_group(label)
613 sgrp = grp.create_group(label)
614 else:
614 else:
615 sgrp = grp
615 sgrp = grp
616 for i in range(dsInfo['dsNumber']):
616 for i in range(dsInfo['dsNumber']):
617 ds = sgrp.create_dataset(
617 ds = sgrp.create_dataset(
618 self.getLabel(dsInfo['variable'], i),
618 self.getLabel(dsInfo['variable'], i),
619 (self.blocksPerFile, ) + dsInfo['shape'][1:],
619 (self.blocksPerFile, ) + dsInfo['shape'][1:],
620 chunks=True,
620 chunks=True,
621 dtype=dsInfo['dtype'])
621 dtype=dsInfo['dtype'])
622 dtsets.append(ds)
622 dtsets.append(ds)
623 data.append((dsInfo['variable'], i))
623 data.append((dsInfo['variable'], i))
624 fp.flush()
624 fp.flush()
625
625
626 log.log('Creating file: {}'.format(fp.filename), self.name)
626 log.log('Creating file: {}'.format(fp.filename), self.name)
627
627
628 self.ds = dtsets
628 self.ds = dtsets
629 self.data = data
629 self.data = data
630 self.firsttime = True
630 self.firsttime = True
631
631
632 return
632 return
633
633
634 def putData(self):
634 def putData(self):
635
635
636 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
636 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
637 self.closeFile()
637 self.closeFile()
638 self.setNextFile()
638 self.setNextFile()
639 self.dataOut.flagNoData = False
639 self.dataOut.flagNoData = False
640 self.blockIndex = 0
640 self.blockIndex = 0
641 return
641 return
642
642
643
643
644
644
645 if self.blockIndex == 0:
645 if self.blockIndex == 0:
646 #Escribir metadata Aqui???
646 #Escribir metadata Aqui???
647 #Setting HDF5 File
647 #Setting HDF5 File
648 self.fp = h5py.File(self.filename, 'w')
648 self.fp = h5py.File(self.filename, 'w')
649 #write metadata
649 #write metadata
650 self.writeMetadata(self.fp)
650 self.writeMetadata(self.fp)
651 #Write data
651 #Write data
652 self.writeData(self.fp)
652 self.writeData(self.fp)
653 log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
653 log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
654 elif (self.blockIndex % 10 ==0):
654 elif (self.blockIndex % 10 ==0):
655 log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
655 log.log('Block No. {}/{} --> {}'.format(self.blockIndex+1, self.blocksPerFile,self.dataOut.datatime.ctime()), self.name)
656 else:
656 else:
657
657
658 log.log('Block No. {}/{}'.format(self.blockIndex+1, self.blocksPerFile), self.name)
658 log.log('Block No. {}/{}'.format(self.blockIndex+1, self.blocksPerFile), self.name)
659
659
660 for i, ds in enumerate(self.ds):
660 for i, ds in enumerate(self.ds):
661 attr, ch = self.data[i]
661 attr, ch = self.data[i]
662 if ch == -1:
662 if ch == -1:
663 ds[self.blockIndex] = getattr(self.dataOut, attr)
663 ds[self.blockIndex] = getattr(self.dataOut, attr)
664 else:
664 else:
665 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
665 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
666
666
667 self.blockIndex += 1
667 self.blockIndex += 1
668
668
669 self.fp.flush()
669 self.fp.flush()
670 self.dataOut.flagNoData = True
670 self.dataOut.flagNoData = True
671
671
672
672
673 def closeFile(self):
673 def closeFile(self):
674
674
675 if self.blockIndex != self.blocksPerFile:
675 if self.blockIndex != self.blocksPerFile:
676 for ds in self.ds:
676 for ds in self.ds:
677 ds.resize(self.blockIndex, axis=0)
677 ds.resize(self.blockIndex, axis=0)
678
678
679 if self.fp:
679 if self.fp:
680 self.fp.flush()
680 self.fp.flush()
681 self.fp.close()
681 self.fp.close()
682
682
683 def close(self):
683 def close(self):
684
684
685 self.closeFile()
685 self.closeFile()
Show file before | Show file after | Hide comments
@@ -1,212 +1,211
1 '''
1 '''
2 Base clases to create Processing units and operations, the MPDecorator
2 Base clases to create Processing units and operations, the MPDecorator
3 must be used in plotting and writing operations to allow to run as an
3 must be used in plotting and writing operations to allow to run as an
4 external process.
4 external process.
5 '''
5 '''
6 import os
6 import os
7 import inspect
7 import inspect
8 import zmq
8 import zmq
9 import time
9 import time
10 import pickle
10 import pickle
11 import traceback
11 import traceback
12 from threading import Thread
12 from threading import Thread
13 from multiprocessing import Process, Queue
13 from multiprocessing import Process, Queue
14 from schainpy.utils import log
14 from schainpy.utils import log
15 import copy
15 import copy
16 QUEUE_SIZE = int(os.environ.get('QUEUE_MAX_SIZE', '10'))
16 QUEUE_SIZE = int(os.environ.get('QUEUE_MAX_SIZE', '10'))
17 class ProcessingUnit(object):
17 class ProcessingUnit(object):
18 '''
18 '''
19 Base class to create Signal Chain Units
19 Base class to create Signal Chain Units
20 '''
20 '''
21
21
22 proc_type = 'processing'
22 proc_type = 'processing'
23
23
24 def __init__(self):
24 def __init__(self):
25
25
26 self.dataIn = None
26 self.dataIn = None
27 self.dataOut = None
27 self.dataOut = None
28 self.isConfig = False
28 self.isConfig = False
29 self.operations = []
29 self.operations = []
30
30
31 def setInput(self, unit):
31 def setInput(self, unit):
32
32
33 self.dataIn = unit.dataOut
33 self.dataIn = unit.dataOut
34
34
35
35
36 def getAllowedArgs(self):
36 def getAllowedArgs(self):
37 if hasattr(self, '__attrs__'):
37 if hasattr(self, '__attrs__'):
38 return self.__attrs__
38 return self.__attrs__
39 else:
39 else:
40 return inspect.getargspec(self.run).args
40 return inspect.getargspec(self.run).args
41
41
42 def addOperation(self, conf, operation):
42 def addOperation(self, conf, operation):
43 '''
43 '''
44 '''
44 '''
45
45
46 self.operations.append((operation, conf.type, conf.getKwargs()))
46 self.operations.append((operation, conf.type, conf.getKwargs()))
47
47
48 def getOperationObj(self, objId):
48 def getOperationObj(self, objId):
49
49
50 if objId not in list(self.operations.keys()):
50 if objId not in list(self.operations.keys()):
51 return None
51 return None
52
52
53 return self.operations[objId]
53 return self.operations[objId]
54
54
55 def call(self, **kwargs):
55 def call(self, **kwargs):
56 '''
56 '''
57 '''
57 '''
58
58
59 try:
59 try:
60 # if self.dataIn is not None and self.dataIn.flagNoData and not self.dataIn.error:
60
61 # return self.dataIn.isReady()
61 if self.dataIn is not None and self.dataIn.flagNoData and not self.dataIn.error:
62 #dataIn=None es unidades de Lectura, segunda parte unidades de procesamiento
62 return self.dataIn.isReady()
63 if self.dataIn is None or (not self.dataIn.error and not self.dataIn.flagNoData):
63 elif self.dataIn is None or not self.dataIn.error:
64 self.run(**kwargs)
64 self.run(**kwargs)
65 elif self.dataIn.error:
65 elif self.dataIn.error:
66 self.dataOut.error = self.dataIn.error
66 self.dataOut.error = self.dataIn.error
67 self.dataOut.flagNoData = True
67 self.dataOut.flagNoData = True
68
69 except:
68 except:
70
69
71 err = traceback.format_exc()
70 err = traceback.format_exc()
72 if 'SchainWarning' in err:
71 if 'SchainWarning' in err:
73 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), self.name)
72 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), self.name)
74 elif 'SchainError' in err:
73 elif 'SchainError' in err:
75 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), self.name)
74 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), self.name)
76 else:
75 else:
77 log.error(err, self.name)
76 log.error(err, self.name)
78 self.dataOut.error = True
77 self.dataOut.error = True
79
78
80 for op, optype, opkwargs in self.operations:
79 for op, optype, opkwargs in self.operations:
81 if optype == 'other' and self.dataOut.isReady():
80 if optype == 'other' and self.dataOut.isReady():
82 try:
81 try:
83 self.dataOut = op.run(self.dataOut, **opkwargs)
82 self.dataOut = op.run(self.dataOut, **opkwargs)
84 except Exception as e:
83 except Exception as e:
85 print(e)
84 print(e)
86 self.dataOut.error = True
85 self.dataOut.error = True
87 return 'Error'
86 return 'Error'
88 elif optype == 'external' and self.dataOut.isReady() :
87 elif optype == 'external' and self.dataOut.isReady() :
89 op.queue.put(copy.deepcopy(self.dataOut))
88 op.queue.put(copy.deepcopy(self.dataOut))
90 elif optype == 'external' and self.dataOut.error:
89 elif optype == 'external' and self.dataOut.error:
91 op.queue.put(copy.deepcopy(self.dataOut))
90 op.queue.put(copy.deepcopy(self.dataOut))
92
91
93 return 'Error' if self.dataOut.error else True#self.dataOut.isReady()
92 return 'Error' if self.dataOut.error else True#self.dataOut.isReady()
94
93
95 def setup(self):
94 def setup(self):
96
95
97 raise NotImplementedError
96 raise NotImplementedError
98
97
99 def run(self):
98 def run(self):
100
99
101 raise NotImplementedError
100 raise NotImplementedError
102
101
103 def close(self):
102 def close(self):
104
103
105 return
104 return
106
105
107
106
108 class Operation(object):
107 class Operation(object):
109
108
110 '''
109 '''
111 '''
110 '''
112
111
113 proc_type = 'operation'
112 proc_type = 'operation'
114
113
115 def __init__(self):
114 def __init__(self):
116
115
117 self.id = None
116 self.id = None
118 self.isConfig = False
117 self.isConfig = False
119
118
120 if not hasattr(self, 'name'):
119 if not hasattr(self, 'name'):
121 self.name = self.__class__.__name__
120 self.name = self.__class__.__name__
122
121
123 def getAllowedArgs(self):
122 def getAllowedArgs(self):
124 if hasattr(self, '__attrs__'):
123 if hasattr(self, '__attrs__'):
125 return self.__attrs__
124 return self.__attrs__
126 else:
125 else:
127 return inspect.getargspec(self.run).args
126 return inspect.getargspec(self.run).args
128
127
129 def setup(self):
128 def setup(self):
130
129
131 self.isConfig = True
130 self.isConfig = True
132
131
133 raise NotImplementedError
132 raise NotImplementedError
134
133
135 def run(self, dataIn, **kwargs):
134 def run(self, dataIn, **kwargs):
136 """
135 """
137 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
136 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
138 atributos del objeto dataIn.
137 atributos del objeto dataIn.
139
138
140 Input:
139 Input:
141
140
142 dataIn : objeto del tipo JROData
141 dataIn : objeto del tipo JROData
143
142
144 Return:
143 Return:
145
144
146 None
145 None
147
146
148 Affected:
147 Affected:
149 __buffer : buffer de recepcion de datos.
148 __buffer : buffer de recepcion de datos.
150
149
151 """
150 """
152 if not self.isConfig:
151 if not self.isConfig:
153 self.setup(**kwargs)
152 self.setup(**kwargs)
154
153
155 raise NotImplementedError
154 raise NotImplementedError
156
155
157 def close(self):
156 def close(self):
158
157
159 return
158 return
160
159
161
160
162 def MPDecorator(BaseClass):
161 def MPDecorator(BaseClass):
163 """
162 """
164 Multiprocessing class decorator
163 Multiprocessing class decorator
165
164
166 This function add multiprocessing features to a BaseClass.
165 This function add multiprocessing features to a BaseClass.
167 """
166 """
168
167
169 class MPClass(BaseClass, Process):
168 class MPClass(BaseClass, Process):
170
169
171 def __init__(self, *args, **kwargs):
170 def __init__(self, *args, **kwargs):
172 super(MPClass, self).__init__()
171 super(MPClass, self).__init__()
173 Process.__init__(self)
172 Process.__init__(self)
174
173
175 self.args = args
174 self.args = args
176 self.kwargs = kwargs
175 self.kwargs = kwargs
177 self.t = time.time()
176 self.t = time.time()
178 self.op_type = 'external'
177 self.op_type = 'external'
179 self.name = BaseClass.__name__
178 self.name = BaseClass.__name__
180 self.__doc__ = BaseClass.__doc__
179 self.__doc__ = BaseClass.__doc__
181
180
182 if 'plot' in self.name.lower() and not self.name.endswith('_'):
181 if 'plot' in self.name.lower() and not self.name.endswith('_'):
183 self.name = '{}{}'.format(self.CODE.upper(), 'Plot')
182 self.name = '{}{}'.format(self.CODE.upper(), 'Plot')
184
183
185 self.start_time = time.time()
184 self.start_time = time.time()
186 self.err_queue = args[3]
185 self.err_queue = args[3]
187 self.queue = Queue(maxsize=QUEUE_SIZE)
186 self.queue = Queue(maxsize=QUEUE_SIZE)
188 self.myrun = BaseClass.run
187 self.myrun = BaseClass.run
189
188
190 def run(self):
189 def run(self):
191
190
192 while True:
191 while True:
193
192
194 dataOut = self.queue.get()
193 dataOut = self.queue.get()
195
194
196 if not dataOut.error:
195 if not dataOut.error:
197 try:
196 try:
198 BaseClass.run(self, dataOut, **self.kwargs)
197 BaseClass.run(self, dataOut, **self.kwargs)
199 except:
198 except:
200 err = traceback.format_exc()
199 err = traceback.format_exc()
201 log.error(err, self.name)
200 log.error(err, self.name)
202 else:
201 else:
203 break
202 break
204
203
205 self.close()
204 self.close()
206
205
207 def close(self):
206 def close(self):
208
207
209 BaseClass.close(self)
208 BaseClass.close(self)
210 log.success('Done...(Time:{:4.2f} secs)'.format(time.time()-self.start_time), self.name)
209 log.success('Done...(Time:{:4.2f} secs)'.format(time.time()-self.start_time), self.name)
211
210
212 return MPClass
211 return MPClass
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,1689 +1,1688
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """Spectra processing Unit and operations
5 """Spectra processing Unit and operations
6
6
7 Here you will find the processing unit `SpectraProc` and several operations
7 Here you will find the processing unit `SpectraProc` and several operations
8 to work with Spectra data type
8 to work with Spectra data type
9 """
9 """
10
10
11 import time
11 import time
12 import itertools
12 import itertools
13
13
14 import numpy
14 import numpy
15 import math
15 import math
16
16
17 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
17 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
18 from schainpy.model.data.jrodata import Spectra
18 from schainpy.model.data.jrodata import Spectra
19 from schainpy.model.data.jrodata import hildebrand_sekhon
19 from schainpy.model.data.jrodata import hildebrand_sekhon
20 from schainpy.utils import log
20 from schainpy.utils import log
21
21
22 from scipy.optimize import curve_fit
22 from scipy.optimize import curve_fit
23
23
24 class SpectraProc(ProcessingUnit):
24 class SpectraProc(ProcessingUnit):
25
25
26 def __init__(self):
26 def __init__(self):
27
27
28 ProcessingUnit.__init__(self)
28 ProcessingUnit.__init__(self)
29
29
30 self.buffer = None
30 self.buffer = None
31 self.firstdatatime = None
31 self.firstdatatime = None
32 self.profIndex = 0
32 self.profIndex = 0
33 self.dataOut = Spectra()
33 self.dataOut = Spectra()
34 self.id_min = None
34 self.id_min = None
35 self.id_max = None
35 self.id_max = None
36 self.setupReq = False #Agregar a todas las unidades de proc
36 self.setupReq = False #Agregar a todas las unidades de proc
37
37
38 def __updateSpecFromVoltage(self):
38 def __updateSpecFromVoltage(self):
39
39
40 self.dataOut.timeZone = self.dataIn.timeZone
40 self.dataOut.timeZone = self.dataIn.timeZone
41 self.dataOut.dstFlag = self.dataIn.dstFlag
41 self.dataOut.dstFlag = self.dataIn.dstFlag
42 self.dataOut.errorCount = self.dataIn.errorCount
42 self.dataOut.errorCount = self.dataIn.errorCount
43 self.dataOut.useLocalTime = self.dataIn.useLocalTime
43 self.dataOut.useLocalTime = self.dataIn.useLocalTime
44 try:
44 try:
45 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
45 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
46 except:
46 except:
47 pass
47 pass
48 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
48 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
49 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
49 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
50 self.dataOut.channelList = self.dataIn.channelList
50 self.dataOut.channelList = self.dataIn.channelList
51 self.dataOut.heightList = self.dataIn.heightList
51 self.dataOut.heightList = self.dataIn.heightList
52 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
52 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
53 self.dataOut.nProfiles = self.dataOut.nFFTPoints
53 self.dataOut.nProfiles = self.dataOut.nFFTPoints
54 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
54 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
55 self.dataOut.utctime = self.firstdatatime
55 self.dataOut.utctime = self.firstdatatime
56 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
56 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
57 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
57 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
58 self.dataOut.flagShiftFFT = False
58 self.dataOut.flagShiftFFT = False
59 self.dataOut.nCohInt = self.dataIn.nCohInt
59 self.dataOut.nCohInt = self.dataIn.nCohInt
60 self.dataOut.nIncohInt = 1
60 self.dataOut.nIncohInt = 1
61 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
61 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
62 self.dataOut.frequency = self.dataIn.frequency
62 self.dataOut.frequency = self.dataIn.frequency
63 self.dataOut.realtime = self.dataIn.realtime
63 self.dataOut.realtime = self.dataIn.realtime
64 self.dataOut.azimuth = self.dataIn.azimuth
64 self.dataOut.azimuth = self.dataIn.azimuth
65 self.dataOut.zenith = self.dataIn.zenith
65 self.dataOut.zenith = self.dataIn.zenith
66 self.dataOut.codeList = self.dataIn.codeList
66 self.dataOut.codeList = self.dataIn.codeList
67 self.dataOut.azimuthList = self.dataIn.azimuthList
67 self.dataOut.azimuthList = self.dataIn.azimuthList
68 self.dataOut.elevationList = self.dataIn.elevationList
68 self.dataOut.elevationList = self.dataIn.elevationList
69
69
70
70
71
72 def __getFft(self):
71 def __getFft(self):
73 """
72 """
74 Convierte valores de Voltaje a Spectra
73 Convierte valores de Voltaje a Spectra
75
74
76 Affected:
75 Affected:
77 self.dataOut.data_spc
76 self.dataOut.data_spc
78 self.dataOut.data_cspc
77 self.dataOut.data_cspc
79 self.dataOut.data_dc
78 self.dataOut.data_dc
80 self.dataOut.heightList
79 self.dataOut.heightList
81 self.profIndex
80 self.profIndex
82 self.buffer
81 self.buffer
83 self.dataOut.flagNoData
82 self.dataOut.flagNoData
84 """
83 """
85 fft_volt = numpy.fft.fft(
84 fft_volt = numpy.fft.fft(
86 self.buffer, n=self.dataOut.nFFTPoints, axis=1)
85 self.buffer, n=self.dataOut.nFFTPoints, axis=1)
87 fft_volt = fft_volt.astype(numpy.dtype('complex'))
86 fft_volt = fft_volt.astype(numpy.dtype('complex'))
88 dc = fft_volt[:, 0, :]
87 dc = fft_volt[:, 0, :]
89
88
90 # calculo de self-spectra
89 # calculo de self-spectra
91 fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
90 fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
92 spc = fft_volt * numpy.conjugate(fft_volt)
91 spc = fft_volt * numpy.conjugate(fft_volt)
93 spc = spc.real
92 spc = spc.real
94
93
95 blocksize = 0
94 blocksize = 0
96 blocksize += dc.size
95 blocksize += dc.size
97 blocksize += spc.size
96 blocksize += spc.size
98
97
99 cspc = None
98 cspc = None
100 pairIndex = 0
99 pairIndex = 0
101 if self.dataOut.pairsList != None:
100 if self.dataOut.pairsList != None:
102 # calculo de cross-spectra
101 # calculo de cross-spectra
103 cspc = numpy.zeros(
102 cspc = numpy.zeros(
104 (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
103 (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
105 for pair in self.dataOut.pairsList:
104 for pair in self.dataOut.pairsList:
106 if pair[0] not in self.dataOut.channelList:
105 if pair[0] not in self.dataOut.channelList:
107 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
106 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
108 str(pair), str(self.dataOut.channelList)))
107 str(pair), str(self.dataOut.channelList)))
109 if pair[1] not in self.dataOut.channelList:
108 if pair[1] not in self.dataOut.channelList:
110 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
109 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
111 str(pair), str(self.dataOut.channelList)))
110 str(pair), str(self.dataOut.channelList)))
112
111
113 cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
112 cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
114 numpy.conjugate(fft_volt[pair[1], :, :])
113 numpy.conjugate(fft_volt[pair[1], :, :])
115 pairIndex += 1
114 pairIndex += 1
116 blocksize += cspc.size
115 blocksize += cspc.size
117
116
118 self.dataOut.data_spc = spc
117 self.dataOut.data_spc = spc
119 self.dataOut.data_cspc = cspc
118 self.dataOut.data_cspc = cspc
120 self.dataOut.data_dc = dc
119 self.dataOut.data_dc = dc
121 self.dataOut.blockSize = blocksize
120 self.dataOut.blockSize = blocksize
122 self.dataOut.flagShiftFFT = False
121 self.dataOut.flagShiftFFT = False
123
122
124 def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False):
123 def run(self, nProfiles=None, nFFTPoints=None, pairsList=None, ippFactor=None, shift_fft=False):
125
124
126 if self.dataIn.type == "Spectra":
125 if self.dataIn.type == "Spectra":
127
126
128 try:
127 try:
129 self.dataOut.copy(self.dataIn)
128 self.dataOut.copy(self.dataIn)
130
129
131 except Exception as e:
130 except Exception as e:
132 print(e)
131 print(e)
133
132
134 if shift_fft:
133 if shift_fft:
135 #desplaza a la derecha en el eje 2 determinadas posiciones
134 #desplaza a la derecha en el eje 2 determinadas posiciones
136 shift = int(self.dataOut.nFFTPoints/2)
135 shift = int(self.dataOut.nFFTPoints/2)
137 self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
136 self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
138
137
139 if self.dataOut.data_cspc is not None:
138 if self.dataOut.data_cspc is not None:
140 #desplaza a la derecha en el eje 2 determinadas posiciones
139 #desplaza a la derecha en el eje 2 determinadas posiciones
141 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
140 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
142 if pairsList:
141 if pairsList:
143 self.__selectPairs(pairsList)
142 self.__selectPairs(pairsList)
144
143
145
144
146 elif self.dataIn.type == "Voltage":
145 elif self.dataIn.type == "Voltage":
147
146
148 self.dataOut.flagNoData = True
147 self.dataOut.flagNoData = True
149
148
150 if nFFTPoints == None:
149 if nFFTPoints == None:
151 raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
150 raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
152
151
153 if nProfiles == None:
152 if nProfiles == None:
154 nProfiles = nFFTPoints
153 nProfiles = nFFTPoints
155
154
156 if ippFactor == None:
155 if ippFactor == None:
157 self.dataOut.ippFactor = 1
156 self.dataOut.ippFactor = 1
158
157
159 self.dataOut.nFFTPoints = nFFTPoints
158 self.dataOut.nFFTPoints = nFFTPoints
160
159
161 if self.buffer is None:
160 if self.buffer is None:
162 self.buffer = numpy.zeros((self.dataIn.nChannels,
161 self.buffer = numpy.zeros((self.dataIn.nChannels,
163 nProfiles,
162 nProfiles,
164 self.dataIn.nHeights),
163 self.dataIn.nHeights),
165 dtype='complex')
164 dtype='complex')
166
165
167 if self.dataIn.flagDataAsBlock:
166 if self.dataIn.flagDataAsBlock:
168 nVoltProfiles = self.dataIn.data.shape[1]
167 nVoltProfiles = self.dataIn.data.shape[1]
169
168
170 if nVoltProfiles == nProfiles:
169 if nVoltProfiles == nProfiles:
171 self.buffer = self.dataIn.data.copy()
170 self.buffer = self.dataIn.data.copy()
172 self.profIndex = nVoltProfiles
171 self.profIndex = nVoltProfiles
173
172
174 elif nVoltProfiles < nProfiles:
173 elif nVoltProfiles < nProfiles:
175
174
176 if self.profIndex == 0:
175 if self.profIndex == 0:
177 self.id_min = 0
176 self.id_min = 0
178 self.id_max = nVoltProfiles
177 self.id_max = nVoltProfiles
179
178
180 self.buffer[:, self.id_min:self.id_max,
179 self.buffer[:, self.id_min:self.id_max,
181 :] = self.dataIn.data
180 :] = self.dataIn.data
182 self.profIndex += nVoltProfiles
181 self.profIndex += nVoltProfiles
183 self.id_min += nVoltProfiles
182 self.id_min += nVoltProfiles
184 self.id_max += nVoltProfiles
183 self.id_max += nVoltProfiles
185 else:
184 else:
186 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
185 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
187 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
186 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
188 self.dataOut.flagNoData = True
187 self.dataOut.flagNoData = True
189 else:
188 else:
190 self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
189 self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
191 self.profIndex += 1
190 self.profIndex += 1
192
191
193 if self.firstdatatime == None:
192 if self.firstdatatime == None:
194 self.firstdatatime = self.dataIn.utctime
193 self.firstdatatime = self.dataIn.utctime
195
194
196 if self.profIndex == nProfiles:
195 if self.profIndex == nProfiles:
197 self.__updateSpecFromVoltage()
196 self.__updateSpecFromVoltage()
198 if pairsList == None:
197 if pairsList == None:
199 self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)]
198 self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)]
200 else:
199 else:
201 self.dataOut.pairsList = pairsList
200 self.dataOut.pairsList = pairsList
202 self.__getFft()
201 self.__getFft()
203 self.dataOut.flagNoData = False
202 self.dataOut.flagNoData = False
204 self.firstdatatime = None
203 self.firstdatatime = None
205 self.profIndex = 0
204 self.profIndex = 0
206 else:
205 else:
207 raise ValueError("The type of input object '%s' is not valid".format(
206 raise ValueError("The type of input object '%s' is not valid".format(
208 self.dataIn.type))
207 self.dataIn.type))
209
208
210 def __selectPairs(self, pairsList):
209 def __selectPairs(self, pairsList):
211
210
212 if not pairsList:
211 if not pairsList:
213 return
212 return
214
213
215 pairs = []
214 pairs = []
216 pairsIndex = []
215 pairsIndex = []
217
216
218 for pair in pairsList:
217 for pair in pairsList:
219 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
218 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
220 continue
219 continue
221 pairs.append(pair)
220 pairs.append(pair)
222 pairsIndex.append(pairs.index(pair))
221 pairsIndex.append(pairs.index(pair))
223
222
224 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
223 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
225 self.dataOut.pairsList = pairs
224 self.dataOut.pairsList = pairs
226
225
227 return
226 return
228
227
229 def selectFFTs(self, minFFT, maxFFT ):
228 def selectFFTs(self, minFFT, maxFFT ):
230 """
229 """
231 Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
230 Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
232 minFFT<= FFT <= maxFFT
231 minFFT<= FFT <= maxFFT
233 """
232 """
234
233
235 if (minFFT > maxFFT):
234 if (minFFT > maxFFT):
236 raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
235 raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
237
236
238 if (minFFT < self.dataOut.getFreqRange()[0]):
237 if (minFFT < self.dataOut.getFreqRange()[0]):
239 minFFT = self.dataOut.getFreqRange()[0]
238 minFFT = self.dataOut.getFreqRange()[0]
240
239
241 if (maxFFT > self.dataOut.getFreqRange()[-1]):
240 if (maxFFT > self.dataOut.getFreqRange()[-1]):
242 maxFFT = self.dataOut.getFreqRange()[-1]
241 maxFFT = self.dataOut.getFreqRange()[-1]
243
242
244 minIndex = 0
243 minIndex = 0
245 maxIndex = 0
244 maxIndex = 0
246 FFTs = self.dataOut.getFreqRange()
245 FFTs = self.dataOut.getFreqRange()
247
246
248 inda = numpy.where(FFTs >= minFFT)
247 inda = numpy.where(FFTs >= minFFT)
249 indb = numpy.where(FFTs <= maxFFT)
248 indb = numpy.where(FFTs <= maxFFT)
250
249
251 try:
250 try:
252 minIndex = inda[0][0]
251 minIndex = inda[0][0]
253 except:
252 except:
254 minIndex = 0
253 minIndex = 0
255
254
256 try:
255 try:
257 maxIndex = indb[0][-1]
256 maxIndex = indb[0][-1]
258 except:
257 except:
259 maxIndex = len(FFTs)
258 maxIndex = len(FFTs)
260
259
261 self.selectFFTsByIndex(minIndex, maxIndex)
260 self.selectFFTsByIndex(minIndex, maxIndex)
262
261
263 return 1
262 return 1
264
263
265 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
264 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
266 newheis = numpy.where(
265 newheis = numpy.where(
267 self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
266 self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
268
267
269 if hei_ref != None:
268 if hei_ref != None:
270 newheis = numpy.where(self.dataOut.heightList > hei_ref)
269 newheis = numpy.where(self.dataOut.heightList > hei_ref)
271
270
272 minIndex = min(newheis[0])
271 minIndex = min(newheis[0])
273 maxIndex = max(newheis[0])
272 maxIndex = max(newheis[0])
274 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
273 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
275 heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
274 heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
276
275
277 # determina indices
276 # determina indices
278 nheis = int(self.dataOut.radarControllerHeaderObj.txB /
277 nheis = int(self.dataOut.radarControllerHeaderObj.txB /
279 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
278 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
280 avg_dB = 10 * \
279 avg_dB = 10 * \
281 numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
280 numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
282 beacon_dB = numpy.sort(avg_dB)[-nheis:]
281 beacon_dB = numpy.sort(avg_dB)[-nheis:]
283 beacon_heiIndexList = []
282 beacon_heiIndexList = []
284 for val in avg_dB.tolist():
283 for val in avg_dB.tolist():
285 if val >= beacon_dB[0]:
284 if val >= beacon_dB[0]:
286 beacon_heiIndexList.append(avg_dB.tolist().index(val))
285 beacon_heiIndexList.append(avg_dB.tolist().index(val))
287
286
288 #data_spc = data_spc[:,:,beacon_heiIndexList]
287 #data_spc = data_spc[:,:,beacon_heiIndexList]
289 data_cspc = None
288 data_cspc = None
290 if self.dataOut.data_cspc is not None:
289 if self.dataOut.data_cspc is not None:
291 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
290 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
292 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
291 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
293
292
294 data_dc = None
293 data_dc = None
295 if self.dataOut.data_dc is not None:
294 if self.dataOut.data_dc is not None:
296 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
295 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
297 #data_dc = data_dc[:,beacon_heiIndexList]
296 #data_dc = data_dc[:,beacon_heiIndexList]
298
297
299 self.dataOut.data_spc = data_spc
298 self.dataOut.data_spc = data_spc
300 self.dataOut.data_cspc = data_cspc
299 self.dataOut.data_cspc = data_cspc
301 self.dataOut.data_dc = data_dc
300 self.dataOut.data_dc = data_dc
302 self.dataOut.heightList = heightList
301 self.dataOut.heightList = heightList
303 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
302 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
304
303
305 return 1
304 return 1
306
305
307 def selectFFTsByIndex(self, minIndex, maxIndex):
306 def selectFFTsByIndex(self, minIndex, maxIndex):
308 """
307 """
309
308
310 """
309 """
311
310
312 if (minIndex < 0) or (minIndex > maxIndex):
311 if (minIndex < 0) or (minIndex > maxIndex):
313 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex))
312 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex))
314
313
315 if (maxIndex >= self.dataOut.nProfiles):
314 if (maxIndex >= self.dataOut.nProfiles):
316 maxIndex = self.dataOut.nProfiles-1
315 maxIndex = self.dataOut.nProfiles-1
317
316
318 #Spectra
317 #Spectra
319 data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:]
318 data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:]
320
319
321 data_cspc = None
320 data_cspc = None
322 if self.dataOut.data_cspc is not None:
321 if self.dataOut.data_cspc is not None:
323 data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:]
322 data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:]
324
323
325 data_dc = None
324 data_dc = None
326 if self.dataOut.data_dc is not None:
325 if self.dataOut.data_dc is not None:
327 data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:]
326 data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:]
328
327
329 self.dataOut.data_spc = data_spc
328 self.dataOut.data_spc = data_spc
330 self.dataOut.data_cspc = data_cspc
329 self.dataOut.data_cspc = data_cspc
331 self.dataOut.data_dc = data_dc
330 self.dataOut.data_dc = data_dc
332
331
333 self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
332 self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
334 self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
333 self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
335 self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
334 self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
336
335
337 return 1
336 return 1
338
337
339 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
338 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
340 # validacion de rango
339 # validacion de rango
341 if minHei == None:
340 if minHei == None:
342 minHei = self.dataOut.heightList[0]
341 minHei = self.dataOut.heightList[0]
343
342
344 if maxHei == None:
343 if maxHei == None:
345 maxHei = self.dataOut.heightList[-1]
344 maxHei = self.dataOut.heightList[-1]
346
345
347 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
346 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
348 print('minHei: %.2f is out of the heights range' % (minHei))
347 print('minHei: %.2f is out of the heights range' % (minHei))
349 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
348 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
350 minHei = self.dataOut.heightList[0]
349 minHei = self.dataOut.heightList[0]
351
350
352 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
351 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
353 print('maxHei: %.2f is out of the heights range' % (maxHei))
352 print('maxHei: %.2f is out of the heights range' % (maxHei))
354 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
353 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
355 maxHei = self.dataOut.heightList[-1]
354 maxHei = self.dataOut.heightList[-1]
356
355
357 # validacion de velocidades
356 # validacion de velocidades
358 velrange = self.dataOut.getVelRange(1)
357 velrange = self.dataOut.getVelRange(1)
359
358
360 if minVel == None:
359 if minVel == None:
361 minVel = velrange[0]
360 minVel = velrange[0]
362
361
363 if maxVel == None:
362 if maxVel == None:
364 maxVel = velrange[-1]
363 maxVel = velrange[-1]
365
364
366 if (minVel < velrange[0]) or (minVel > maxVel):
365 if (minVel < velrange[0]) or (minVel > maxVel):
367 print('minVel: %.2f is out of the velocity range' % (minVel))
366 print('minVel: %.2f is out of the velocity range' % (minVel))
368 print('minVel is setting to %.2f' % (velrange[0]))
367 print('minVel is setting to %.2f' % (velrange[0]))
369 minVel = velrange[0]
368 minVel = velrange[0]
370
369
371 if (maxVel > velrange[-1]) or (maxVel < minVel):
370 if (maxVel > velrange[-1]) or (maxVel < minVel):
372 print('maxVel: %.2f is out of the velocity range' % (maxVel))
371 print('maxVel: %.2f is out of the velocity range' % (maxVel))
373 print('maxVel is setting to %.2f' % (velrange[-1]))
372 print('maxVel is setting to %.2f' % (velrange[-1]))
374 maxVel = velrange[-1]
373 maxVel = velrange[-1]
375
374
376 # seleccion de indices para rango
375 # seleccion de indices para rango
377 minIndex = 0
376 minIndex = 0
378 maxIndex = 0
377 maxIndex = 0
379 heights = self.dataOut.heightList
378 heights = self.dataOut.heightList
380
379
381 inda = numpy.where(heights >= minHei)
380 inda = numpy.where(heights >= minHei)
382 indb = numpy.where(heights <= maxHei)
381 indb = numpy.where(heights <= maxHei)
383
382
384 try:
383 try:
385 minIndex = inda[0][0]
384 minIndex = inda[0][0]
386 except:
385 except:
387 minIndex = 0
386 minIndex = 0
388
387
389 try:
388 try:
390 maxIndex = indb[0][-1]
389 maxIndex = indb[0][-1]
391 except:
390 except:
392 maxIndex = len(heights)
391 maxIndex = len(heights)
393
392
394 if (minIndex < 0) or (minIndex > maxIndex):
393 if (minIndex < 0) or (minIndex > maxIndex):
395 raise ValueError("some value in (%d,%d) is not valid" % (
394 raise ValueError("some value in (%d,%d) is not valid" % (
396 minIndex, maxIndex))
395 minIndex, maxIndex))
397
396
398 if (maxIndex >= self.dataOut.nHeights):
397 if (maxIndex >= self.dataOut.nHeights):
399 maxIndex = self.dataOut.nHeights - 1
398 maxIndex = self.dataOut.nHeights - 1
400
399
401 # seleccion de indices para velocidades
400 # seleccion de indices para velocidades
402 indminvel = numpy.where(velrange >= minVel)
401 indminvel = numpy.where(velrange >= minVel)
403 indmaxvel = numpy.where(velrange <= maxVel)
402 indmaxvel = numpy.where(velrange <= maxVel)
404 try:
403 try:
405 minIndexVel = indminvel[0][0]
404 minIndexVel = indminvel[0][0]
406 except:
405 except:
407 minIndexVel = 0
406 minIndexVel = 0
408
407
409 try:
408 try:
410 maxIndexVel = indmaxvel[0][-1]
409 maxIndexVel = indmaxvel[0][-1]
411 except:
410 except:
412 maxIndexVel = len(velrange)
411 maxIndexVel = len(velrange)
413
412
414 # seleccion del espectro
413 # seleccion del espectro
415 data_spc = self.dataOut.data_spc[:,
414 data_spc = self.dataOut.data_spc[:,
416 minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
415 minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
417 # estimacion de ruido
416 # estimacion de ruido
418 noise = numpy.zeros(self.dataOut.nChannels)
417 noise = numpy.zeros(self.dataOut.nChannels)
419
418
420 for channel in range(self.dataOut.nChannels):
419 for channel in range(self.dataOut.nChannels):
421 daux = data_spc[channel, :, :]
420 daux = data_spc[channel, :, :]
422 sortdata = numpy.sort(daux, axis=None)
421 sortdata = numpy.sort(daux, axis=None)
423 noise[channel] = hildebrand_sekhon(sortdata, self.dataOut.nIncohInt)
422 noise[channel] = hildebrand_sekhon(sortdata, self.dataOut.nIncohInt)
424
423
425 self.dataOut.noise_estimation = noise.copy()
424 self.dataOut.noise_estimation = noise.copy()
426
425
427 return 1
426 return 1
428
427
429 class removeDC(Operation):
428 class removeDC(Operation):
430
429
431 def run(self, dataOut, mode=2):
430 def run(self, dataOut, mode=2):
432 self.dataOut = dataOut
431 self.dataOut = dataOut
433 jspectra = self.dataOut.data_spc
432 jspectra = self.dataOut.data_spc
434 jcspectra = self.dataOut.data_cspc
433 jcspectra = self.dataOut.data_cspc
435
434
436 num_chan = jspectra.shape[0]
435 num_chan = jspectra.shape[0]
437 num_hei = jspectra.shape[2]
436 num_hei = jspectra.shape[2]
438
437
439 if jcspectra is not None:
438 if jcspectra is not None:
440 jcspectraExist = True
439 jcspectraExist = True
441 num_pairs = jcspectra.shape[0]
440 num_pairs = jcspectra.shape[0]
442 else:
441 else:
443 jcspectraExist = False
442 jcspectraExist = False
444
443
445 freq_dc = int(jspectra.shape[1] / 2)
444 freq_dc = int(jspectra.shape[1] / 2)
446 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
445 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
447 ind_vel = ind_vel.astype(int)
446 ind_vel = ind_vel.astype(int)
448
447
449 if ind_vel[0] < 0:
448 if ind_vel[0] < 0:
450 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
449 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
451
450
452 if mode == 1:
451 if mode == 1:
453 jspectra[:, freq_dc, :] = (
452 jspectra[:, freq_dc, :] = (
454 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
453 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
455
454
456 if jcspectraExist:
455 if jcspectraExist:
457 jcspectra[:, freq_dc, :] = (
456 jcspectra[:, freq_dc, :] = (
458 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
457 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
459
458
460 if mode == 2:
459 if mode == 2:
461
460
462 vel = numpy.array([-2, -1, 1, 2])
461 vel = numpy.array([-2, -1, 1, 2])
463 xx = numpy.zeros([4, 4])
462 xx = numpy.zeros([4, 4])
464
463
465 for fil in range(4):
464 for fil in range(4):
466 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
465 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
467
466
468 xx_inv = numpy.linalg.inv(xx)
467 xx_inv = numpy.linalg.inv(xx)
469 xx_aux = xx_inv[0, :]
468 xx_aux = xx_inv[0, :]
470
469
471 for ich in range(num_chan):
470 for ich in range(num_chan):
472 yy = jspectra[ich, ind_vel, :]
471 yy = jspectra[ich, ind_vel, :]
473 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
472 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
474
473
475 junkid = jspectra[ich, freq_dc, :] <= 0
474 junkid = jspectra[ich, freq_dc, :] <= 0
476 cjunkid = sum(junkid)
475 cjunkid = sum(junkid)
477
476
478 if cjunkid.any():
477 if cjunkid.any():
479 jspectra[ich, freq_dc, junkid.nonzero()] = (
478 jspectra[ich, freq_dc, junkid.nonzero()] = (
480 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
479 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
481
480
482 if jcspectraExist:
481 if jcspectraExist:
483 for ip in range(num_pairs):
482 for ip in range(num_pairs):
484 yy = jcspectra[ip, ind_vel, :]
483 yy = jcspectra[ip, ind_vel, :]
485 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
484 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
486
485
487 self.dataOut.data_spc = jspectra
486 self.dataOut.data_spc = jspectra
488 self.dataOut.data_cspc = jcspectra
487 self.dataOut.data_cspc = jcspectra
489
488
490 return self.dataOut
489 return self.dataOut
491
490
492 # import matplotlib.pyplot as plt
491 # import matplotlib.pyplot as plt
493
492
494 def fit_func( x, a0, a1, a2): #, a3, a4, a5):
493 def fit_func( x, a0, a1, a2): #, a3, a4, a5):
495 z = (x - a1) / a2
494 z = (x - a1) / a2
496 y = a0 * numpy.exp(-z**2 / a2) #+ a3 + a4 * x + a5 * x**2
495 y = a0 * numpy.exp(-z**2 / a2) #+ a3 + a4 * x + a5 * x**2
497 return y
496 return y
498
497
499
498
500 class CleanRayleigh(Operation):
499 class CleanRayleigh(Operation):
501
500
502 def __init__(self):
501 def __init__(self):
503
502
504 Operation.__init__(self)
503 Operation.__init__(self)
505 self.i=0
504 self.i=0
506 self.isConfig = False
505 self.isConfig = False
507 self.__dataReady = False
506 self.__dataReady = False
508 self.__profIndex = 0
507 self.__profIndex = 0
509 self.byTime = False
508 self.byTime = False
510 self.byProfiles = False
509 self.byProfiles = False
511
510
512 self.bloques = None
511 self.bloques = None
513 self.bloque0 = None
512 self.bloque0 = None
514
513
515 self.index = 0
514 self.index = 0
516
515
517 self.buffer = 0
516 self.buffer = 0
518 self.buffer2 = 0
517 self.buffer2 = 0
519 self.buffer3 = 0
518 self.buffer3 = 0
520
519
521
520
522 def setup(self,dataOut,min_hei,max_hei,n, timeInterval,factor_stdv):
521 def setup(self,dataOut,min_hei,max_hei,n, timeInterval,factor_stdv):
523
522
524 self.nChannels = dataOut.nChannels
523 self.nChannels = dataOut.nChannels
525 self.nProf = dataOut.nProfiles
524 self.nProf = dataOut.nProfiles
526 self.nPairs = dataOut.data_cspc.shape[0]
525 self.nPairs = dataOut.data_cspc.shape[0]
527 self.pairsArray = numpy.array(dataOut.pairsList)
526 self.pairsArray = numpy.array(dataOut.pairsList)
528 self.spectra = dataOut.data_spc
527 self.spectra = dataOut.data_spc
529 self.cspectra = dataOut.data_cspc
528 self.cspectra = dataOut.data_cspc
530 self.heights = dataOut.heightList #alturas totales
529 self.heights = dataOut.heightList #alturas totales
531 self.nHeights = len(self.heights)
530 self.nHeights = len(self.heights)
532 self.min_hei = min_hei
531 self.min_hei = min_hei
533 self.max_hei = max_hei
532 self.max_hei = max_hei
534 if (self.min_hei == None):
533 if (self.min_hei == None):
535 self.min_hei = 0
534 self.min_hei = 0
536 if (self.max_hei == None):
535 if (self.max_hei == None):
537 self.max_hei = dataOut.heightList[-1]
536 self.max_hei = dataOut.heightList[-1]
538 self.hval = ((self.max_hei>=self.heights) & (self.heights >= self.min_hei)).nonzero()
537 self.hval = ((self.max_hei>=self.heights) & (self.heights >= self.min_hei)).nonzero()
539 self.heightsClean = self.heights[self.hval] #alturas filtradas
538 self.heightsClean = self.heights[self.hval] #alturas filtradas
540 self.hval = self.hval[0] # forma (N,), an solo N elementos -> Indices de alturas
539 self.hval = self.hval[0] # forma (N,), an solo N elementos -> Indices de alturas
541 self.nHeightsClean = len(self.heightsClean)
540 self.nHeightsClean = len(self.heightsClean)
542 self.channels = dataOut.channelList
541 self.channels = dataOut.channelList
543 self.nChan = len(self.channels)
542 self.nChan = len(self.channels)
544 self.nIncohInt = dataOut.nIncohInt
543 self.nIncohInt = dataOut.nIncohInt
545 self.__initime = dataOut.utctime
544 self.__initime = dataOut.utctime
546 self.maxAltInd = self.hval[-1]+1
545 self.maxAltInd = self.hval[-1]+1
547 self.minAltInd = self.hval[0]
546 self.minAltInd = self.hval[0]
548
547
549 self.crosspairs = dataOut.pairsList
548 self.crosspairs = dataOut.pairsList
550 self.nPairs = len(self.crosspairs)
549 self.nPairs = len(self.crosspairs)
551 self.normFactor = dataOut.normFactor
550 self.normFactor = dataOut.normFactor
552 self.nFFTPoints = dataOut.nFFTPoints
551 self.nFFTPoints = dataOut.nFFTPoints
553 self.ippSeconds = dataOut.ippSeconds
552 self.ippSeconds = dataOut.ippSeconds
554 self.currentTime = self.__initime
553 self.currentTime = self.__initime
555 self.pairsArray = numpy.array(dataOut.pairsList)
554 self.pairsArray = numpy.array(dataOut.pairsList)
556 self.factor_stdv = factor_stdv
555 self.factor_stdv = factor_stdv
557
556
558 if n != None :
557 if n != None :
559 self.byProfiles = True
558 self.byProfiles = True
560 self.nIntProfiles = n
559 self.nIntProfiles = n
561 else:
560 else:
562 self.__integrationtime = timeInterval
561 self.__integrationtime = timeInterval
563
562
564 self.__dataReady = False
563 self.__dataReady = False
565 self.isConfig = True
564 self.isConfig = True
566
565
567
566
568
567
569 def run(self, dataOut,min_hei=None,max_hei=None, n=None, timeInterval=10,factor_stdv=2.5):
568 def run(self, dataOut,min_hei=None,max_hei=None, n=None, timeInterval=10,factor_stdv=2.5):
570
569
571 if not self.isConfig :
570 if not self.isConfig :
572
571
573 self.setup(dataOut, min_hei,max_hei,n,timeInterval,factor_stdv)
572 self.setup(dataOut, min_hei,max_hei,n,timeInterval,factor_stdv)
574
573
575 tini=dataOut.utctime
574 tini=dataOut.utctime
576
575
577 if self.byProfiles:
576 if self.byProfiles:
578 if self.__profIndex == self.nIntProfiles:
577 if self.__profIndex == self.nIntProfiles:
579 self.__dataReady = True
578 self.__dataReady = True
580 else:
579 else:
581 if (tini - self.__initime) >= self.__integrationtime:
580 if (tini - self.__initime) >= self.__integrationtime:
582
581
583 self.__dataReady = True
582 self.__dataReady = True
584 self.__initime = tini
583 self.__initime = tini
585
584
586 #if (tini.tm_min % 2) == 0 and (tini.tm_sec < 5 and self.fint==0):
585 #if (tini.tm_min % 2) == 0 and (tini.tm_sec < 5 and self.fint==0):
587
586
588 if self.__dataReady:
587 if self.__dataReady:
589
588
590 self.__profIndex = 0
589 self.__profIndex = 0
591 jspc = self.buffer
590 jspc = self.buffer
592 jcspc = self.buffer2
591 jcspc = self.buffer2
593 #jnoise = self.buffer3
592 #jnoise = self.buffer3
594 self.buffer = dataOut.data_spc
593 self.buffer = dataOut.data_spc
595 self.buffer2 = dataOut.data_cspc
594 self.buffer2 = dataOut.data_cspc
596 #self.buffer3 = dataOut.noise
595 #self.buffer3 = dataOut.noise
597 self.currentTime = dataOut.utctime
596 self.currentTime = dataOut.utctime
598 if numpy.any(jspc) :
597 if numpy.any(jspc) :
599 #print( jspc.shape, jcspc.shape)
598 #print( jspc.shape, jcspc.shape)
600 jspc = numpy.reshape(jspc,(int(len(jspc)/self.nChannels),self.nChannels,self.nFFTPoints,self.nHeights))
599 jspc = numpy.reshape(jspc,(int(len(jspc)/self.nChannels),self.nChannels,self.nFFTPoints,self.nHeights))
601 jcspc= numpy.reshape(jcspc,(int(len(jcspc)/self.nPairs),self.nPairs,self.nFFTPoints,self.nHeights))
600 jcspc= numpy.reshape(jcspc,(int(len(jcspc)/self.nPairs),self.nPairs,self.nFFTPoints,self.nHeights))
602 self.__dataReady = False
601 self.__dataReady = False
603 #print( jspc.shape, jcspc.shape)
602 #print( jspc.shape, jcspc.shape)
604 dataOut.flagNoData = False
603 dataOut.flagNoData = False
605 else:
604 else:
606 dataOut.flagNoData = True
605 dataOut.flagNoData = True
607 self.__dataReady = False
606 self.__dataReady = False
608 return dataOut
607 return dataOut
609 else:
608 else:
610 #print( len(self.buffer))
609 #print( len(self.buffer))
611 if numpy.any(self.buffer):
610 if numpy.any(self.buffer):
612 self.buffer = numpy.concatenate((self.buffer,dataOut.data_spc), axis=0)
611 self.buffer = numpy.concatenate((self.buffer,dataOut.data_spc), axis=0)
613 self.buffer2 = numpy.concatenate((self.buffer2,dataOut.data_cspc), axis=0)
612 self.buffer2 = numpy.concatenate((self.buffer2,dataOut.data_cspc), axis=0)
614 self.buffer3 += dataOut.data_dc
613 self.buffer3 += dataOut.data_dc
615 else:
614 else:
616 self.buffer = dataOut.data_spc
615 self.buffer = dataOut.data_spc
617 self.buffer2 = dataOut.data_cspc
616 self.buffer2 = dataOut.data_cspc
618 self.buffer3 = dataOut.data_dc
617 self.buffer3 = dataOut.data_dc
619 #print self.index, self.fint
618 #print self.index, self.fint
620 #print self.buffer2.shape
619 #print self.buffer2.shape
621 dataOut.flagNoData = True ## NOTE: ?? revisar LUEGO
620 dataOut.flagNoData = True ## NOTE: ?? revisar LUEGO
622 self.__profIndex += 1
621 self.__profIndex += 1
623 return dataOut ## NOTE: REV
622 return dataOut ## NOTE: REV
624
623
625
624
626 #index = tini.tm_hour*12+tini.tm_min/5
625 #index = tini.tm_hour*12+tini.tm_min/5
627 '''REVISAR'''
626 '''REVISAR'''
628 # jspc = jspc/self.nFFTPoints/self.normFactor
627 # jspc = jspc/self.nFFTPoints/self.normFactor
629 # jcspc = jcspc/self.nFFTPoints/self.normFactor
628 # jcspc = jcspc/self.nFFTPoints/self.normFactor
630
629
631
630
632
631
633 tmp_spectra,tmp_cspectra = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
632 tmp_spectra,tmp_cspectra = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
634 dataOut.data_spc = tmp_spectra
633 dataOut.data_spc = tmp_spectra
635 dataOut.data_cspc = tmp_cspectra
634 dataOut.data_cspc = tmp_cspectra
636
635
637 #dataOut.data_spc,dataOut.data_cspc = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
636 #dataOut.data_spc,dataOut.data_cspc = self.cleanRayleigh(dataOut,jspc,jcspc,self.factor_stdv)
638
637
639 dataOut.data_dc = self.buffer3
638 dataOut.data_dc = self.buffer3
640 dataOut.nIncohInt *= self.nIntProfiles
639 dataOut.nIncohInt *= self.nIntProfiles
641 dataOut.utctime = self.currentTime #tiempo promediado
640 dataOut.utctime = self.currentTime #tiempo promediado
642 #print("Time: ",time.localtime(dataOut.utctime))
641 #print("Time: ",time.localtime(dataOut.utctime))
643 # dataOut.data_spc = sat_spectra
642 # dataOut.data_spc = sat_spectra
644 # dataOut.data_cspc = sat_cspectra
643 # dataOut.data_cspc = sat_cspectra
645 self.buffer = 0
644 self.buffer = 0
646 self.buffer2 = 0
645 self.buffer2 = 0
647 self.buffer3 = 0
646 self.buffer3 = 0
648
647
649 return dataOut
648 return dataOut
650
649
651 def cleanRayleigh(self,dataOut,spectra,cspectra,factor_stdv):
650 def cleanRayleigh(self,dataOut,spectra,cspectra,factor_stdv):
652 #print("OP cleanRayleigh")
651 #print("OP cleanRayleigh")
653 #import matplotlib.pyplot as plt
652 #import matplotlib.pyplot as plt
654 #for k in range(149):
653 #for k in range(149):
655 #channelsProcssd = []
654 #channelsProcssd = []
656 #channelA_ok = False
655 #channelA_ok = False
657 #rfunc = cspectra.copy() #self.bloques
656 #rfunc = cspectra.copy() #self.bloques
658 rfunc = spectra.copy()
657 rfunc = spectra.copy()
659 #rfunc = cspectra
658 #rfunc = cspectra
660 #val_spc = spectra*0.0 #self.bloque0*0.0
659 #val_spc = spectra*0.0 #self.bloque0*0.0
661 #val_cspc = cspectra*0.0 #self.bloques*0.0
660 #val_cspc = cspectra*0.0 #self.bloques*0.0
662 #in_sat_spectra = spectra.copy() #self.bloque0
661 #in_sat_spectra = spectra.copy() #self.bloque0
663 #in_sat_cspectra = cspectra.copy() #self.bloques
662 #in_sat_cspectra = cspectra.copy() #self.bloques
664
663
665
664
666 ###ONLY FOR TEST:
665 ###ONLY FOR TEST:
667 raxs = math.ceil(math.sqrt(self.nPairs))
666 raxs = math.ceil(math.sqrt(self.nPairs))
668 caxs = math.ceil(self.nPairs/raxs)
667 caxs = math.ceil(self.nPairs/raxs)
669 if self.nPairs <4:
668 if self.nPairs <4:
670 raxs = 2
669 raxs = 2
671 caxs = 2
670 caxs = 2
672 #print(raxs, caxs)
671 #print(raxs, caxs)
673 fft_rev = 14 #nFFT to plot
672 fft_rev = 14 #nFFT to plot
674 hei_rev = ((self.heights >= 550) & (self.heights <= 551)).nonzero() #hei to plot
673 hei_rev = ((self.heights >= 550) & (self.heights <= 551)).nonzero() #hei to plot
675 hei_rev = hei_rev[0]
674 hei_rev = hei_rev[0]
676 #print(hei_rev)
675 #print(hei_rev)
677
676
678 #print numpy.absolute(rfunc[:,0,0,14])
677 #print numpy.absolute(rfunc[:,0,0,14])
679
678
680 gauss_fit, covariance = None, None
679 gauss_fit, covariance = None, None
681 for ih in range(self.minAltInd,self.maxAltInd):
680 for ih in range(self.minAltInd,self.maxAltInd):
682 for ifreq in range(self.nFFTPoints):
681 for ifreq in range(self.nFFTPoints):
683 '''
682 '''
684 ###ONLY FOR TEST:
683 ###ONLY FOR TEST:
685 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
684 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
686 fig, axs = plt.subplots(raxs, caxs)
685 fig, axs = plt.subplots(raxs, caxs)
687 fig2, axs2 = plt.subplots(raxs, caxs)
686 fig2, axs2 = plt.subplots(raxs, caxs)
688 col_ax = 0
687 col_ax = 0
689 row_ax = 0
688 row_ax = 0
690 '''
689 '''
691 #print(self.nPairs)
690 #print(self.nPairs)
692 for ii in range(self.nChan): #PARES DE CANALES SELF y CROSS
691 for ii in range(self.nChan): #PARES DE CANALES SELF y CROSS
693 # if self.crosspairs[ii][1]-self.crosspairs[ii][0] > 1: # APLICAR SOLO EN PARES CONTIGUOS
692 # if self.crosspairs[ii][1]-self.crosspairs[ii][0] > 1: # APLICAR SOLO EN PARES CONTIGUOS
694 # continue
693 # continue
695 # if not self.crosspairs[ii][0] in channelsProcssd:
694 # if not self.crosspairs[ii][0] in channelsProcssd:
696 # channelA_ok = True
695 # channelA_ok = True
697 #print("pair: ",self.crosspairs[ii])
696 #print("pair: ",self.crosspairs[ii])
698 '''
697 '''
699 ###ONLY FOR TEST:
698 ###ONLY FOR TEST:
700 if (col_ax%caxs==0 and col_ax!=0 and self.nPairs !=1):
699 if (col_ax%caxs==0 and col_ax!=0 and self.nPairs !=1):
701 col_ax = 0
700 col_ax = 0
702 row_ax += 1
701 row_ax += 1
703 '''
702 '''
704 func2clean = 10*numpy.log10(numpy.absolute(rfunc[:,ii,ifreq,ih])) #Potencia?
703 func2clean = 10*numpy.log10(numpy.absolute(rfunc[:,ii,ifreq,ih])) #Potencia?
705 #print(func2clean.shape)
704 #print(func2clean.shape)
706 val = (numpy.isfinite(func2clean)==True).nonzero()
705 val = (numpy.isfinite(func2clean)==True).nonzero()
707
706
708 if len(val)>0: #limitador
707 if len(val)>0: #limitador
709 min_val = numpy.around(numpy.amin(func2clean)-2) #> (-40)
708 min_val = numpy.around(numpy.amin(func2clean)-2) #> (-40)
710 if min_val <= -40 :
709 if min_val <= -40 :
711 min_val = -40
710 min_val = -40
712 max_val = numpy.around(numpy.amax(func2clean)+2) #< 200
711 max_val = numpy.around(numpy.amax(func2clean)+2) #< 200
713 if max_val >= 200 :
712 if max_val >= 200 :
714 max_val = 200
713 max_val = 200
715 #print min_val, max_val
714 #print min_val, max_val
716 step = 1
715 step = 1
717 #print("Getting bins and the histogram")
716 #print("Getting bins and the histogram")
718 x_dist = min_val + numpy.arange(1 + ((max_val-(min_val))/step))*step
717 x_dist = min_val + numpy.arange(1 + ((max_val-(min_val))/step))*step
719 y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
718 y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
720 #print(len(y_dist),len(binstep[:-1]))
719 #print(len(y_dist),len(binstep[:-1]))
721 #print(row_ax,col_ax, " ..")
720 #print(row_ax,col_ax, " ..")
722 #print(self.pairsArray[ii][0],self.pairsArray[ii][1])
721 #print(self.pairsArray[ii][0],self.pairsArray[ii][1])
723 mean = numpy.sum(x_dist * y_dist) / numpy.sum(y_dist)
722 mean = numpy.sum(x_dist * y_dist) / numpy.sum(y_dist)
724 sigma = numpy.sqrt(numpy.sum(y_dist * (x_dist - mean)**2) / numpy.sum(y_dist))
723 sigma = numpy.sqrt(numpy.sum(y_dist * (x_dist - mean)**2) / numpy.sum(y_dist))
725 parg = [numpy.amax(y_dist),mean,sigma]
724 parg = [numpy.amax(y_dist),mean,sigma]
726
725
727 newY = None
726 newY = None
728
727
729 try :
728 try :
730 gauss_fit, covariance = curve_fit(fit_func, x_dist, y_dist,p0=parg)
729 gauss_fit, covariance = curve_fit(fit_func, x_dist, y_dist,p0=parg)
731 mode = gauss_fit[1]
730 mode = gauss_fit[1]
732 stdv = gauss_fit[2]
731 stdv = gauss_fit[2]
733 #print(" FIT OK",gauss_fit)
732 #print(" FIT OK",gauss_fit)
734 '''
733 '''
735 ###ONLY FOR TEST:
734 ###ONLY FOR TEST:
736 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
735 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
737 newY = fit_func(x_dist,gauss_fit[0],gauss_fit[1],gauss_fit[2])
736 newY = fit_func(x_dist,gauss_fit[0],gauss_fit[1],gauss_fit[2])
738 axs[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
737 axs[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
739 axs[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
738 axs[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
740 axs[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
739 axs[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
741 '''
740 '''
742 except:
741 except:
743 mode = mean
742 mode = mean
744 stdv = sigma
743 stdv = sigma
745 #print("FIT FAIL")
744 #print("FIT FAIL")
746 #continue
745 #continue
747
746
748
747
749 #print(mode,stdv)
748 #print(mode,stdv)
750 #Removing echoes greater than mode + std_factor*stdv
749 #Removing echoes greater than mode + std_factor*stdv
751 noval = (abs(func2clean - mode)>=(factor_stdv*stdv)).nonzero()
750 noval = (abs(func2clean - mode)>=(factor_stdv*stdv)).nonzero()
752 #noval tiene los indices que se van a remover
751 #noval tiene los indices que se van a remover
753 #print("Chan ",ii," novals: ",len(noval[0]))
752 #print("Chan ",ii," novals: ",len(noval[0]))
754 if len(noval[0]) > 0: #forma de array (N,) es igual a longitud (N)
753 if len(noval[0]) > 0: #forma de array (N,) es igual a longitud (N)
755 novall = ((func2clean - mode) >= (factor_stdv*stdv)).nonzero()
754 novall = ((func2clean - mode) >= (factor_stdv*stdv)).nonzero()
756 #print(novall)
755 #print(novall)
757 #print(" ",self.pairsArray[ii])
756 #print(" ",self.pairsArray[ii])
758 #cross_pairs = self.pairsArray[ii]
757 #cross_pairs = self.pairsArray[ii]
759 #Getting coherent echoes which are removed.
758 #Getting coherent echoes which are removed.
760 # if len(novall[0]) > 0:
759 # if len(novall[0]) > 0:
761 #
760 #
762 # val_spc[novall[0],cross_pairs[0],ifreq,ih] = 1
761 # val_spc[novall[0],cross_pairs[0],ifreq,ih] = 1
763 # val_spc[novall[0],cross_pairs[1],ifreq,ih] = 1
762 # val_spc[novall[0],cross_pairs[1],ifreq,ih] = 1
764 # val_cspc[novall[0],ii,ifreq,ih] = 1
763 # val_cspc[novall[0],ii,ifreq,ih] = 1
765 #print("OUT NOVALL 1")
764 #print("OUT NOVALL 1")
766 try:
765 try:
767 pair = (self.channels[ii],self.channels[ii + 1])
766 pair = (self.channels[ii],self.channels[ii + 1])
768 except:
767 except:
769 pair = (99,99)
768 pair = (99,99)
770 #print("par ", pair)
769 #print("par ", pair)
771 if ( pair in self.crosspairs):
770 if ( pair in self.crosspairs):
772 q = self.crosspairs.index(pair)
771 q = self.crosspairs.index(pair)
773 #print("estΓ‘ aqui: ", q, (ii,ii + 1))
772 #print("estΓ‘ aqui: ", q, (ii,ii + 1))
774 new_a = numpy.delete(cspectra[:,q,ifreq,ih], noval[0])
773 new_a = numpy.delete(cspectra[:,q,ifreq,ih], noval[0])
775 cspectra[noval,q,ifreq,ih] = numpy.mean(new_a) #mean CrossSpectra
774 cspectra[noval,q,ifreq,ih] = numpy.mean(new_a) #mean CrossSpectra
776
775
777 #if channelA_ok:
776 #if channelA_ok:
778 #chA = self.channels.index(cross_pairs[0])
777 #chA = self.channels.index(cross_pairs[0])
779 new_b = numpy.delete(spectra[:,ii,ifreq,ih], noval[0])
778 new_b = numpy.delete(spectra[:,ii,ifreq,ih], noval[0])
780 spectra[noval,ii,ifreq,ih] = numpy.mean(new_b) #mean Spectra Pair A
779 spectra[noval,ii,ifreq,ih] = numpy.mean(new_b) #mean Spectra Pair A
781 #channelA_ok = False
780 #channelA_ok = False
782
781
783 # chB = self.channels.index(cross_pairs[1])
782 # chB = self.channels.index(cross_pairs[1])
784 # new_c = numpy.delete(spectra[:,chB,ifreq,ih], noval[0])
783 # new_c = numpy.delete(spectra[:,chB,ifreq,ih], noval[0])
785 # spectra[noval,chB,ifreq,ih] = numpy.mean(new_c) #mean Spectra Pair B
784 # spectra[noval,chB,ifreq,ih] = numpy.mean(new_c) #mean Spectra Pair B
786 #
785 #
787 # channelsProcssd.append(self.crosspairs[ii][0]) # save channel A
786 # channelsProcssd.append(self.crosspairs[ii][0]) # save channel A
788 # channelsProcssd.append(self.crosspairs[ii][1]) # save channel B
787 # channelsProcssd.append(self.crosspairs[ii][1]) # save channel B
789 '''
788 '''
790 ###ONLY FOR TEST:
789 ###ONLY FOR TEST:
791 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
790 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
792 func2clean = 10*numpy.log10(numpy.absolute(spectra[:,ii,ifreq,ih]))
791 func2clean = 10*numpy.log10(numpy.absolute(spectra[:,ii,ifreq,ih]))
793 y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
792 y_dist,binstep = numpy.histogram(func2clean,bins=range(int(min_val),int(max_val+2),step))
794 axs2[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
793 axs2[row_ax,col_ax].plot(binstep[:-1],newY,color='red')
795 axs2[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
794 axs2[row_ax,col_ax].plot(binstep[:-1],y_dist,color='green')
796 axs2[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
795 axs2[row_ax,col_ax].set_title("CH "+str(self.channels[ii]))
797 '''
796 '''
798 '''
797 '''
799 ###ONLY FOR TEST:
798 ###ONLY FOR TEST:
800 col_ax += 1 #contador de ploteo columnas
799 col_ax += 1 #contador de ploteo columnas
801 ##print(col_ax)
800 ##print(col_ax)
802 ###ONLY FOR TEST:
801 ###ONLY FOR TEST:
803 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
802 if ifreq ==fft_rev and ih==hei_rev: #TO VIEW A SIGNLE FREQUENCY
804 title = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km"
803 title = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km"
805 title2 = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km CLEANED"
804 title2 = str(dataOut.datatime)+" nFFT: "+str(ifreq)+" Alt: "+str(self.heights[ih])+ " km CLEANED"
806 fig.suptitle(title)
805 fig.suptitle(title)
807 fig2.suptitle(title2)
806 fig2.suptitle(title2)
808 plt.show()
807 plt.show()
809 '''
808 '''
810 ##################################################################################################
809 ##################################################################################################
811
810
812 #print("Getting average of the spectra and cross-spectra from incoherent echoes.")
811 #print("Getting average of the spectra and cross-spectra from incoherent echoes.")
813 out_spectra = numpy.zeros([self.nChan,self.nFFTPoints,self.nHeights], dtype=float) #+numpy.nan
812 out_spectra = numpy.zeros([self.nChan,self.nFFTPoints,self.nHeights], dtype=float) #+numpy.nan
814 out_cspectra = numpy.zeros([self.nPairs,self.nFFTPoints,self.nHeights], dtype=complex) #+numpy.nan
813 out_cspectra = numpy.zeros([self.nPairs,self.nFFTPoints,self.nHeights], dtype=complex) #+numpy.nan
815 for ih in range(self.nHeights):
814 for ih in range(self.nHeights):
816 for ifreq in range(self.nFFTPoints):
815 for ifreq in range(self.nFFTPoints):
817 for ich in range(self.nChan):
816 for ich in range(self.nChan):
818 tmp = spectra[:,ich,ifreq,ih]
817 tmp = spectra[:,ich,ifreq,ih]
819 valid = (numpy.isfinite(tmp[:])==True).nonzero()
818 valid = (numpy.isfinite(tmp[:])==True).nonzero()
820
819
821 if len(valid[0]) >0 :
820 if len(valid[0]) >0 :
822 out_spectra[ich,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
821 out_spectra[ich,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
823
822
824 for icr in range(self.nPairs):
823 for icr in range(self.nPairs):
825 tmp = numpy.squeeze(cspectra[:,icr,ifreq,ih])
824 tmp = numpy.squeeze(cspectra[:,icr,ifreq,ih])
826 valid = (numpy.isfinite(tmp)==True).nonzero()
825 valid = (numpy.isfinite(tmp)==True).nonzero()
827 if len(valid[0]) > 0:
826 if len(valid[0]) > 0:
828 out_cspectra[icr,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
827 out_cspectra[icr,ifreq,ih] = numpy.nansum(tmp)#/len(valid[0])
829
828
830 return out_spectra, out_cspectra
829 return out_spectra, out_cspectra
831
830
832 def REM_ISOLATED_POINTS(self,array,rth):
831 def REM_ISOLATED_POINTS(self,array,rth):
833 # import matplotlib.pyplot as plt
832 # import matplotlib.pyplot as plt
834 if rth == None :
833 if rth == None :
835 rth = 4
834 rth = 4
836 #print("REM ISO")
835 #print("REM ISO")
837 num_prof = len(array[0,:,0])
836 num_prof = len(array[0,:,0])
838 num_hei = len(array[0,0,:])
837 num_hei = len(array[0,0,:])
839 n2d = len(array[:,0,0])
838 n2d = len(array[:,0,0])
840
839
841 for ii in range(n2d) :
840 for ii in range(n2d) :
842 #print ii,n2d
841 #print ii,n2d
843 tmp = array[ii,:,:]
842 tmp = array[ii,:,:]
844 #print tmp.shape, array[ii,101,:],array[ii,102,:]
843 #print tmp.shape, array[ii,101,:],array[ii,102,:]
845
844
846 # fig = plt.figure(figsize=(6,5))
845 # fig = plt.figure(figsize=(6,5))
847 # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
846 # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
848 # ax = fig.add_axes([left, bottom, width, height])
847 # ax = fig.add_axes([left, bottom, width, height])
849 # x = range(num_prof)
848 # x = range(num_prof)
850 # y = range(num_hei)
849 # y = range(num_hei)
851 # cp = ax.contour(y,x,tmp)
850 # cp = ax.contour(y,x,tmp)
852 # ax.clabel(cp, inline=True,fontsize=10)
851 # ax.clabel(cp, inline=True,fontsize=10)
853 # plt.show()
852 # plt.show()
854
853
855 #indxs = WHERE(FINITE(tmp) AND tmp GT 0,cindxs)
854 #indxs = WHERE(FINITE(tmp) AND tmp GT 0,cindxs)
856 tmp = numpy.reshape(tmp,num_prof*num_hei)
855 tmp = numpy.reshape(tmp,num_prof*num_hei)
857 indxs1 = (numpy.isfinite(tmp)==True).nonzero()
856 indxs1 = (numpy.isfinite(tmp)==True).nonzero()
858 indxs2 = (tmp > 0).nonzero()
857 indxs2 = (tmp > 0).nonzero()
859
858
860 indxs1 = (indxs1[0])
859 indxs1 = (indxs1[0])
861 indxs2 = indxs2[0]
860 indxs2 = indxs2[0]
862 #indxs1 = numpy.array(indxs1[0])
861 #indxs1 = numpy.array(indxs1[0])
863 #indxs2 = numpy.array(indxs2[0])
862 #indxs2 = numpy.array(indxs2[0])
864 indxs = None
863 indxs = None
865 #print indxs1 , indxs2
864 #print indxs1 , indxs2
866 for iv in range(len(indxs2)):
865 for iv in range(len(indxs2)):
867 indv = numpy.array((indxs1 == indxs2[iv]).nonzero())
866 indv = numpy.array((indxs1 == indxs2[iv]).nonzero())
868 #print len(indxs2), indv
867 #print len(indxs2), indv
869 if len(indv[0]) > 0 :
868 if len(indv[0]) > 0 :
870 indxs = numpy.concatenate((indxs,indxs2[iv]), axis=None)
869 indxs = numpy.concatenate((indxs,indxs2[iv]), axis=None)
871 # print indxs
870 # print indxs
872 indxs = indxs[1:]
871 indxs = indxs[1:]
873 #print(indxs, len(indxs))
872 #print(indxs, len(indxs))
874 if len(indxs) < 4 :
873 if len(indxs) < 4 :
875 array[ii,:,:] = 0.
874 array[ii,:,:] = 0.
876 return
875 return
877
876
878 xpos = numpy.mod(indxs ,num_hei)
877 xpos = numpy.mod(indxs ,num_hei)
879 ypos = (indxs / num_hei)
878 ypos = (indxs / num_hei)
880 sx = numpy.argsort(xpos) # Ordering respect to "x" (time)
879 sx = numpy.argsort(xpos) # Ordering respect to "x" (time)
881 #print sx
880 #print sx
882 xpos = xpos[sx]
881 xpos = xpos[sx]
883 ypos = ypos[sx]
882 ypos = ypos[sx]
884
883
885 # *********************************** Cleaning isolated points **********************************
884 # *********************************** Cleaning isolated points **********************************
886 ic = 0
885 ic = 0
887 while True :
886 while True :
888 r = numpy.sqrt(list(numpy.power((xpos[ic]-xpos),2)+ numpy.power((ypos[ic]-ypos),2)))
887 r = numpy.sqrt(list(numpy.power((xpos[ic]-xpos),2)+ numpy.power((ypos[ic]-ypos),2)))
889 #no_coh = WHERE(FINITE(r) AND (r LE rth),cno_coh)
888 #no_coh = WHERE(FINITE(r) AND (r LE rth),cno_coh)
890 #plt.plot(r)
889 #plt.plot(r)
891 #plt.show()
890 #plt.show()
892 no_coh1 = (numpy.isfinite(r)==True).nonzero()
891 no_coh1 = (numpy.isfinite(r)==True).nonzero()
893 no_coh2 = (r <= rth).nonzero()
892 no_coh2 = (r <= rth).nonzero()
894 #print r, no_coh1, no_coh2
893 #print r, no_coh1, no_coh2
895 no_coh1 = numpy.array(no_coh1[0])
894 no_coh1 = numpy.array(no_coh1[0])
896 no_coh2 = numpy.array(no_coh2[0])
895 no_coh2 = numpy.array(no_coh2[0])
897 no_coh = None
896 no_coh = None
898 #print valid1 , valid2
897 #print valid1 , valid2
899 for iv in range(len(no_coh2)):
898 for iv in range(len(no_coh2)):
900 indv = numpy.array((no_coh1 == no_coh2[iv]).nonzero())
899 indv = numpy.array((no_coh1 == no_coh2[iv]).nonzero())
901 if len(indv[0]) > 0 :
900 if len(indv[0]) > 0 :
902 no_coh = numpy.concatenate((no_coh,no_coh2[iv]), axis=None)
901 no_coh = numpy.concatenate((no_coh,no_coh2[iv]), axis=None)
903 no_coh = no_coh[1:]
902 no_coh = no_coh[1:]
904 #print len(no_coh), no_coh
903 #print len(no_coh), no_coh
905 if len(no_coh) < 4 :
904 if len(no_coh) < 4 :
906 #print xpos[ic], ypos[ic], ic
905 #print xpos[ic], ypos[ic], ic
907 # plt.plot(r)
906 # plt.plot(r)
908 # plt.show()
907 # plt.show()
909 xpos[ic] = numpy.nan
908 xpos[ic] = numpy.nan
910 ypos[ic] = numpy.nan
909 ypos[ic] = numpy.nan
911
910
912 ic = ic + 1
911 ic = ic + 1
913 if (ic == len(indxs)) :
912 if (ic == len(indxs)) :
914 break
913 break
915 #print( xpos, ypos)
914 #print( xpos, ypos)
916
915
917 indxs = (numpy.isfinite(list(xpos))==True).nonzero()
916 indxs = (numpy.isfinite(list(xpos))==True).nonzero()
918 #print indxs[0]
917 #print indxs[0]
919 if len(indxs[0]) < 4 :
918 if len(indxs[0]) < 4 :
920 array[ii,:,:] = 0.
919 array[ii,:,:] = 0.
921 return
920 return
922
921
923 xpos = xpos[indxs[0]]
922 xpos = xpos[indxs[0]]
924 ypos = ypos[indxs[0]]
923 ypos = ypos[indxs[0]]
925 for i in range(0,len(ypos)):
924 for i in range(0,len(ypos)):
926 ypos[i]=int(ypos[i])
925 ypos[i]=int(ypos[i])
927 junk = tmp
926 junk = tmp
928 tmp = junk*0.0
927 tmp = junk*0.0
929
928
930 tmp[list(xpos + (ypos*num_hei))] = junk[list(xpos + (ypos*num_hei))]
929 tmp[list(xpos + (ypos*num_hei))] = junk[list(xpos + (ypos*num_hei))]
931 array[ii,:,:] = numpy.reshape(tmp,(num_prof,num_hei))
930 array[ii,:,:] = numpy.reshape(tmp,(num_prof,num_hei))
932
931
933 #print array.shape
932 #print array.shape
934 #tmp = numpy.reshape(tmp,(num_prof,num_hei))
933 #tmp = numpy.reshape(tmp,(num_prof,num_hei))
935 #print tmp.shape
934 #print tmp.shape
936
935
937 # fig = plt.figure(figsize=(6,5))
936 # fig = plt.figure(figsize=(6,5))
938 # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
937 # left, bottom, width, height = 0.1, 0.1, 0.8, 0.8
939 # ax = fig.add_axes([left, bottom, width, height])
938 # ax = fig.add_axes([left, bottom, width, height])
940 # x = range(num_prof)
939 # x = range(num_prof)
941 # y = range(num_hei)
940 # y = range(num_hei)
942 # cp = ax.contour(y,x,array[ii,:,:])
941 # cp = ax.contour(y,x,array[ii,:,:])
943 # ax.clabel(cp, inline=True,fontsize=10)
942 # ax.clabel(cp, inline=True,fontsize=10)
944 # plt.show()
943 # plt.show()
945 return array
944 return array
946
945
947
946
948 class IntegrationFaradaySpectra(Operation):
947 class IntegrationFaradaySpectra(Operation):
949
948
950 __profIndex = 0
949 __profIndex = 0
951 __withOverapping = False
950 __withOverapping = False
952
951
953 __byTime = False
952 __byTime = False
954 __initime = None
953 __initime = None
955 __lastdatatime = None
954 __lastdatatime = None
956 __integrationtime = None
955 __integrationtime = None
957
956
958 __buffer_spc = None
957 __buffer_spc = None
959 __buffer_cspc = None
958 __buffer_cspc = None
960 __buffer_dc = None
959 __buffer_dc = None
961
960
962 __dataReady = False
961 __dataReady = False
963
962
964 __timeInterval = None
963 __timeInterval = None
965
964
966 n = None
965 n = None
967
966
968 def __init__(self):
967 def __init__(self):
969
968
970 Operation.__init__(self)
969 Operation.__init__(self)
971
970
972 def setup(self, dataOut,n=None, timeInterval=None, overlapping=False, DPL=None):
971 def setup(self, dataOut,n=None, timeInterval=None, overlapping=False, DPL=None):
973 """
972 """
974 Set the parameters of the integration class.
973 Set the parameters of the integration class.
975
974
976 Inputs:
975 Inputs:
977
976
978 n : Number of coherent integrations
977 n : Number of coherent integrations
979 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
978 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
980 overlapping :
979 overlapping :
981
980
982 """
981 """
983
982
984 self.__initime = None
983 self.__initime = None
985 self.__lastdatatime = 0
984 self.__lastdatatime = 0
986
985
987 self.__buffer_spc = []
986 self.__buffer_spc = []
988 self.__buffer_cspc = []
987 self.__buffer_cspc = []
989 self.__buffer_dc = 0
988 self.__buffer_dc = 0
990
989
991 self.__profIndex = 0
990 self.__profIndex = 0
992 self.__dataReady = False
991 self.__dataReady = False
993 self.__byTime = False
992 self.__byTime = False
994
993
995 #self.ByLags = dataOut.ByLags ###REDEFINIR
994 #self.ByLags = dataOut.ByLags ###REDEFINIR
996 self.ByLags = False
995 self.ByLags = False
997
996
998 if DPL != None:
997 if DPL != None:
999 self.DPL=DPL
998 self.DPL=DPL
1000 else:
999 else:
1001 #self.DPL=dataOut.DPL ###REDEFINIR
1000 #self.DPL=dataOut.DPL ###REDEFINIR
1002 self.DPL=0
1001 self.DPL=0
1003
1002
1004 if n is None and timeInterval is None:
1003 if n is None and timeInterval is None:
1005 raise ValueError("n or timeInterval should be specified ...")
1004 raise ValueError("n or timeInterval should be specified ...")
1006
1005
1007 if n is not None:
1006 if n is not None:
1008 self.n = int(n)
1007 self.n = int(n)
1009 else:
1008 else:
1010
1009
1011 self.__integrationtime = int(timeInterval)
1010 self.__integrationtime = int(timeInterval)
1012 self.n = None
1011 self.n = None
1013 self.__byTime = True
1012 self.__byTime = True
1014
1013
1015 def putData(self, data_spc, data_cspc, data_dc):
1014 def putData(self, data_spc, data_cspc, data_dc):
1016 """
1015 """
1017 Add a profile to the __buffer_spc and increase in one the __profileIndex
1016 Add a profile to the __buffer_spc and increase in one the __profileIndex
1018
1017
1019 """
1018 """
1020
1019
1021 self.__buffer_spc.append(data_spc)
1020 self.__buffer_spc.append(data_spc)
1022
1021
1023 if data_cspc is None:
1022 if data_cspc is None:
1024 self.__buffer_cspc = None
1023 self.__buffer_cspc = None
1025 else:
1024 else:
1026 self.__buffer_cspc.append(data_cspc)
1025 self.__buffer_cspc.append(data_cspc)
1027
1026
1028 if data_dc is None:
1027 if data_dc is None:
1029 self.__buffer_dc = None
1028 self.__buffer_dc = None
1030 else:
1029 else:
1031 self.__buffer_dc += data_dc
1030 self.__buffer_dc += data_dc
1032
1031
1033 self.__profIndex += 1
1032 self.__profIndex += 1
1034
1033
1035 return
1034 return
1036
1035
1037 def hildebrand_sekhon_Integration(self,data,navg):
1036 def hildebrand_sekhon_Integration(self,data,navg):
1038
1037
1039 sortdata = numpy.sort(data, axis=None)
1038 sortdata = numpy.sort(data, axis=None)
1040 sortID=data.argsort()
1039 sortID=data.argsort()
1041 lenOfData = len(sortdata)
1040 lenOfData = len(sortdata)
1042 nums_min = lenOfData*0.75
1041 nums_min = lenOfData*0.75
1043 if nums_min <= 5:
1042 if nums_min <= 5:
1044 nums_min = 5
1043 nums_min = 5
1045 sump = 0.
1044 sump = 0.
1046 sumq = 0.
1045 sumq = 0.
1047 j = 0
1046 j = 0
1048 cont = 1
1047 cont = 1
1049 while((cont == 1)and(j < lenOfData)):
1048 while((cont == 1)and(j < lenOfData)):
1050 sump += sortdata[j]
1049 sump += sortdata[j]
1051 sumq += sortdata[j]**2
1050 sumq += sortdata[j]**2
1052 if j > nums_min:
1051 if j > nums_min:
1053 rtest = float(j)/(j-1) + 1.0/navg
1052 rtest = float(j)/(j-1) + 1.0/navg
1054 if ((sumq*j) > (rtest*sump**2)):
1053 if ((sumq*j) > (rtest*sump**2)):
1055 j = j - 1
1054 j = j - 1
1056 sump = sump - sortdata[j]
1055 sump = sump - sortdata[j]
1057 sumq = sumq - sortdata[j]**2
1056 sumq = sumq - sortdata[j]**2
1058 cont = 0
1057 cont = 0
1059 j += 1
1058 j += 1
1060 #lnoise = sump / j
1059 #lnoise = sump / j
1061
1060
1062 return j,sortID
1061 return j,sortID
1063
1062
1064 def pushData(self):
1063 def pushData(self):
1065 """
1064 """
1066 Return the sum of the last profiles and the profiles used in the sum.
1065 Return the sum of the last profiles and the profiles used in the sum.
1067
1066
1068 Affected:
1067 Affected:
1069
1068
1070 self.__profileIndex
1069 self.__profileIndex
1071
1070
1072 """
1071 """
1073 bufferH=None
1072 bufferH=None
1074 buffer=None
1073 buffer=None
1075 buffer1=None
1074 buffer1=None
1076 buffer_cspc=None
1075 buffer_cspc=None
1077 self.__buffer_spc=numpy.array(self.__buffer_spc)
1076 self.__buffer_spc=numpy.array(self.__buffer_spc)
1078 self.__buffer_cspc=numpy.array(self.__buffer_cspc)
1077 self.__buffer_cspc=numpy.array(self.__buffer_cspc)
1079 freq_dc = int(self.__buffer_spc.shape[2] / 2)
1078 freq_dc = int(self.__buffer_spc.shape[2] / 2)
1080 #print("FREQ_DC",freq_dc,self.__buffer_spc.shape,self.nHeights)
1079 #print("FREQ_DC",freq_dc,self.__buffer_spc.shape,self.nHeights)
1081 for k in range(7,self.nHeights):
1080 for k in range(7,self.nHeights):
1082 buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k])
1081 buffer_cspc=numpy.copy(self.__buffer_cspc[:,:,:,k])
1083 outliers_IDs_cspc=[]
1082 outliers_IDs_cspc=[]
1084 cspc_outliers_exist=False
1083 cspc_outliers_exist=False
1085 for i in range(self.nChannels):#dataOut.nChannels):
1084 for i in range(self.nChannels):#dataOut.nChannels):
1086
1085
1087 buffer1=numpy.copy(self.__buffer_spc[:,i,:,k])
1086 buffer1=numpy.copy(self.__buffer_spc[:,i,:,k])
1088 indexes=[]
1087 indexes=[]
1089 #sortIDs=[]
1088 #sortIDs=[]
1090 outliers_IDs=[]
1089 outliers_IDs=[]
1091
1090
1092 for j in range(self.nProfiles):
1091 for j in range(self.nProfiles):
1093 # if i==0 and j==freq_dc: #NOT CONSIDERING DC PROFILE AT CHANNEL 0
1092 # if i==0 and j==freq_dc: #NOT CONSIDERING DC PROFILE AT CHANNEL 0
1094 # continue
1093 # continue
1095 # if i==1 and j==0: #NOT CONSIDERING DC PROFILE AT CHANNEL 1
1094 # if i==1 and j==0: #NOT CONSIDERING DC PROFILE AT CHANNEL 1
1096 # continue
1095 # continue
1097 buffer=buffer1[:,j]
1096 buffer=buffer1[:,j]
1098 index,sortID=self.hildebrand_sekhon_Integration(buffer,1)
1097 index,sortID=self.hildebrand_sekhon_Integration(buffer,1)
1099
1098
1100 indexes.append(index)
1099 indexes.append(index)
1101 #sortIDs.append(sortID)
1100 #sortIDs.append(sortID)
1102 outliers_IDs=numpy.append(outliers_IDs,sortID[index:])
1101 outliers_IDs=numpy.append(outliers_IDs,sortID[index:])
1103
1102
1104 outliers_IDs=numpy.array(outliers_IDs)
1103 outliers_IDs=numpy.array(outliers_IDs)
1105 outliers_IDs=outliers_IDs.ravel()
1104 outliers_IDs=outliers_IDs.ravel()
1106 outliers_IDs=numpy.unique(outliers_IDs)
1105 outliers_IDs=numpy.unique(outliers_IDs)
1107 outliers_IDs=outliers_IDs.astype(numpy.dtype('int64'))
1106 outliers_IDs=outliers_IDs.astype(numpy.dtype('int64'))
1108 indexes=numpy.array(indexes)
1107 indexes=numpy.array(indexes)
1109 indexmin=numpy.min(indexes)
1108 indexmin=numpy.min(indexes)
1110
1109
1111 if indexmin != buffer1.shape[0]:
1110 if indexmin != buffer1.shape[0]:
1112 cspc_outliers_exist=True
1111 cspc_outliers_exist=True
1113 ###sortdata=numpy.sort(buffer1,axis=0)
1112 ###sortdata=numpy.sort(buffer1,axis=0)
1114 ###avg2=numpy.mean(sortdata[:indexmin,:],axis=0)
1113 ###avg2=numpy.mean(sortdata[:indexmin,:],axis=0)
1115 lt=outliers_IDs
1114 lt=outliers_IDs
1116 avg=numpy.mean(buffer1[[t for t in range(buffer1.shape[0]) if t not in lt],:],axis=0)
1115 avg=numpy.mean(buffer1[[t for t in range(buffer1.shape[0]) if t not in lt],:],axis=0)
1117
1116
1118 for p in list(outliers_IDs):
1117 for p in list(outliers_IDs):
1119 buffer1[p,:]=avg
1118 buffer1[p,:]=avg
1120
1119
1121 self.__buffer_spc[:,i,:,k]=numpy.copy(buffer1)
1120 self.__buffer_spc[:,i,:,k]=numpy.copy(buffer1)
1122 ###cspc IDs
1121 ###cspc IDs
1123 #indexmin_cspc+=indexmin_cspc
1122 #indexmin_cspc+=indexmin_cspc
1124 outliers_IDs_cspc=numpy.append(outliers_IDs_cspc,outliers_IDs)
1123 outliers_IDs_cspc=numpy.append(outliers_IDs_cspc,outliers_IDs)
1125
1124
1126 #if not breakFlag:
1125 #if not breakFlag:
1127 outliers_IDs_cspc=outliers_IDs_cspc.astype(numpy.dtype('int64'))
1126 outliers_IDs_cspc=outliers_IDs_cspc.astype(numpy.dtype('int64'))
1128 if cspc_outliers_exist:
1127 if cspc_outliers_exist:
1129 #sortdata=numpy.sort(buffer_cspc,axis=0)
1128 #sortdata=numpy.sort(buffer_cspc,axis=0)
1130 #avg=numpy.mean(sortdata[:indexmin_cpsc,:],axis=0)
1129 #avg=numpy.mean(sortdata[:indexmin_cpsc,:],axis=0)
1131 lt=outliers_IDs_cspc
1130 lt=outliers_IDs_cspc
1132
1131
1133 avg=numpy.mean(buffer_cspc[[t for t in range(buffer_cspc.shape[0]) if t not in lt],:],axis=0)
1132 avg=numpy.mean(buffer_cspc[[t for t in range(buffer_cspc.shape[0]) if t not in lt],:],axis=0)
1134 for p in list(outliers_IDs_cspc):
1133 for p in list(outliers_IDs_cspc):
1135 buffer_cspc[p,:]=avg
1134 buffer_cspc[p,:]=avg
1136
1135
1137 self.__buffer_cspc[:,:,:,k]=numpy.copy(buffer_cspc)
1136 self.__buffer_cspc[:,:,:,k]=numpy.copy(buffer_cspc)
1138 #else:
1137 #else:
1139 #break
1138 #break
1140
1139
1141
1140
1142
1141
1143
1142
1144 buffer=None
1143 buffer=None
1145 bufferH=None
1144 bufferH=None
1146 buffer1=None
1145 buffer1=None
1147 buffer_cspc=None
1146 buffer_cspc=None
1148
1147
1149 #print("cpsc",self.__buffer_cspc[:,0,0,0,0])
1148 #print("cpsc",self.__buffer_cspc[:,0,0,0,0])
1150 #print(self.__profIndex)
1149 #print(self.__profIndex)
1151 #exit()
1150 #exit()
1152
1151
1153 buffer=None
1152 buffer=None
1154 #print(self.__buffer_spc[:,1,3,20,0])
1153 #print(self.__buffer_spc[:,1,3,20,0])
1155 #print(self.__buffer_spc[:,1,5,37,0])
1154 #print(self.__buffer_spc[:,1,5,37,0])
1156 data_spc = numpy.sum(self.__buffer_spc,axis=0)
1155 data_spc = numpy.sum(self.__buffer_spc,axis=0)
1157 data_cspc = numpy.sum(self.__buffer_cspc,axis=0)
1156 data_cspc = numpy.sum(self.__buffer_cspc,axis=0)
1158
1157
1159 #print(numpy.shape(data_spc))
1158 #print(numpy.shape(data_spc))
1160 #data_spc[1,4,20,0]=numpy.nan
1159 #data_spc[1,4,20,0]=numpy.nan
1161
1160
1162 #data_cspc = self.__buffer_cspc
1161 #data_cspc = self.__buffer_cspc
1163 data_dc = self.__buffer_dc
1162 data_dc = self.__buffer_dc
1164 n = self.__profIndex
1163 n = self.__profIndex
1165
1164
1166 self.__buffer_spc = []
1165 self.__buffer_spc = []
1167 self.__buffer_cspc = []
1166 self.__buffer_cspc = []
1168 self.__buffer_dc = 0
1167 self.__buffer_dc = 0
1169 self.__profIndex = 0
1168 self.__profIndex = 0
1170
1169
1171 return data_spc, data_cspc, data_dc, n
1170 return data_spc, data_cspc, data_dc, n
1172
1171
1173 def byProfiles(self, *args):
1172 def byProfiles(self, *args):
1174
1173
1175 self.__dataReady = False
1174 self.__dataReady = False
1176 avgdata_spc = None
1175 avgdata_spc = None
1177 avgdata_cspc = None
1176 avgdata_cspc = None
1178 avgdata_dc = None
1177 avgdata_dc = None
1179
1178
1180 self.putData(*args)
1179 self.putData(*args)
1181
1180
1182 if self.__profIndex == self.n:
1181 if self.__profIndex == self.n:
1183
1182
1184 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1183 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1185 self.n = n
1184 self.n = n
1186 self.__dataReady = True
1185 self.__dataReady = True
1187
1186
1188 return avgdata_spc, avgdata_cspc, avgdata_dc
1187 return avgdata_spc, avgdata_cspc, avgdata_dc
1189
1188
1190 def byTime(self, datatime, *args):
1189 def byTime(self, datatime, *args):
1191
1190
1192 self.__dataReady = False
1191 self.__dataReady = False
1193 avgdata_spc = None
1192 avgdata_spc = None
1194 avgdata_cspc = None
1193 avgdata_cspc = None
1195 avgdata_dc = None
1194 avgdata_dc = None
1196
1195
1197 self.putData(*args)
1196 self.putData(*args)
1198
1197
1199 if (datatime - self.__initime) >= self.__integrationtime:
1198 if (datatime - self.__initime) >= self.__integrationtime:
1200 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1199 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1201 self.n = n
1200 self.n = n
1202 self.__dataReady = True
1201 self.__dataReady = True
1203
1202
1204 return avgdata_spc, avgdata_cspc, avgdata_dc
1203 return avgdata_spc, avgdata_cspc, avgdata_dc
1205
1204
1206 def integrate(self, datatime, *args):
1205 def integrate(self, datatime, *args):
1207
1206
1208 if self.__profIndex == 0:
1207 if self.__profIndex == 0:
1209 self.__initime = datatime
1208 self.__initime = datatime
1210
1209
1211 if self.__byTime:
1210 if self.__byTime:
1212 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1211 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1213 datatime, *args)
1212 datatime, *args)
1214 else:
1213 else:
1215 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1214 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1216
1215
1217 if not self.__dataReady:
1216 if not self.__dataReady:
1218 return None, None, None, None
1217 return None, None, None, None
1219
1218
1220 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1219 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1221
1220
1222 def run(self, dataOut, n=None, DPL = None,timeInterval=None, overlapping=False):
1221 def run(self, dataOut, n=None, DPL = None,timeInterval=None, overlapping=False):
1223 if n == 1:
1222 if n == 1:
1224 return dataOut
1223 return dataOut
1225
1224
1226 dataOut.flagNoData = True
1225 dataOut.flagNoData = True
1227
1226
1228 if not self.isConfig:
1227 if not self.isConfig:
1229 self.setup(dataOut, n, timeInterval, overlapping,DPL )
1228 self.setup(dataOut, n, timeInterval, overlapping,DPL )
1230 self.isConfig = True
1229 self.isConfig = True
1231
1230
1232 if not self.ByLags:
1231 if not self.ByLags:
1233 self.nProfiles=dataOut.nProfiles
1232 self.nProfiles=dataOut.nProfiles
1234 self.nChannels=dataOut.nChannels
1233 self.nChannels=dataOut.nChannels
1235 self.nHeights=dataOut.nHeights
1234 self.nHeights=dataOut.nHeights
1236 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
1235 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
1237 dataOut.data_spc,
1236 dataOut.data_spc,
1238 dataOut.data_cspc,
1237 dataOut.data_cspc,
1239 dataOut.data_dc)
1238 dataOut.data_dc)
1240 else:
1239 else:
1241 self.nProfiles=dataOut.nProfiles
1240 self.nProfiles=dataOut.nProfiles
1242 self.nChannels=dataOut.nChannels
1241 self.nChannels=dataOut.nChannels
1243 self.nHeights=dataOut.nHeights
1242 self.nHeights=dataOut.nHeights
1244 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
1243 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
1245 dataOut.dataLag_spc,
1244 dataOut.dataLag_spc,
1246 dataOut.dataLag_cspc,
1245 dataOut.dataLag_cspc,
1247 dataOut.dataLag_dc)
1246 dataOut.dataLag_dc)
1248
1247
1249 if self.__dataReady:
1248 if self.__dataReady:
1250
1249
1251 if not self.ByLags:
1250 if not self.ByLags:
1252
1251
1253 dataOut.data_spc = numpy.squeeze(avgdata_spc)
1252 dataOut.data_spc = numpy.squeeze(avgdata_spc)
1254 dataOut.data_cspc = numpy.squeeze(avgdata_cspc)
1253 dataOut.data_cspc = numpy.squeeze(avgdata_cspc)
1255 dataOut.data_dc = avgdata_dc
1254 dataOut.data_dc = avgdata_dc
1256 else:
1255 else:
1257 dataOut.dataLag_spc = avgdata_spc
1256 dataOut.dataLag_spc = avgdata_spc
1258 dataOut.dataLag_cspc = avgdata_cspc
1257 dataOut.dataLag_cspc = avgdata_cspc
1259 dataOut.dataLag_dc = avgdata_dc
1258 dataOut.dataLag_dc = avgdata_dc
1260
1259
1261 dataOut.data_spc=dataOut.dataLag_spc[:,:,:,dataOut.LagPlot]
1260 dataOut.data_spc=dataOut.dataLag_spc[:,:,:,dataOut.LagPlot]
1262 dataOut.data_cspc=dataOut.dataLag_cspc[:,:,:,dataOut.LagPlot]
1261 dataOut.data_cspc=dataOut.dataLag_cspc[:,:,:,dataOut.LagPlot]
1263 dataOut.data_dc=dataOut.dataLag_dc[:,:,dataOut.LagPlot]
1262 dataOut.data_dc=dataOut.dataLag_dc[:,:,dataOut.LagPlot]
1264
1263
1265
1264
1266 dataOut.nIncohInt *= self.n
1265 dataOut.nIncohInt *= self.n
1267 dataOut.utctime = avgdatatime
1266 dataOut.utctime = avgdatatime
1268 dataOut.flagNoData = False
1267 dataOut.flagNoData = False
1269
1268
1270 return dataOut
1269 return dataOut
1271
1270
1272 class removeInterference(Operation):
1271 class removeInterference(Operation):
1273
1272
1274 def removeInterference2(self):
1273 def removeInterference2(self):
1275
1274
1276 cspc = self.dataOut.data_cspc
1275 cspc = self.dataOut.data_cspc
1277 spc = self.dataOut.data_spc
1276 spc = self.dataOut.data_spc
1278 Heights = numpy.arange(cspc.shape[2])
1277 Heights = numpy.arange(cspc.shape[2])
1279 realCspc = numpy.abs(cspc)
1278 realCspc = numpy.abs(cspc)
1280
1279
1281 for i in range(cspc.shape[0]):
1280 for i in range(cspc.shape[0]):
1282 LinePower= numpy.sum(realCspc[i], axis=0)
1281 LinePower= numpy.sum(realCspc[i], axis=0)
1283 Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
1282 Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
1284 SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
1283 SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
1285 InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
1284 InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
1286 InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
1285 InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
1287 InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
1286 InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
1288
1287
1289
1288
1290 InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
1289 InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
1291 #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
1290 #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
1292 if len(InterferenceRange)<int(cspc.shape[1]*0.3):
1291 if len(InterferenceRange)<int(cspc.shape[1]*0.3):
1293 cspc[i,InterferenceRange,:] = numpy.NaN
1292 cspc[i,InterferenceRange,:] = numpy.NaN
1294
1293
1295 self.dataOut.data_cspc = cspc
1294 self.dataOut.data_cspc = cspc
1296
1295
1297 def removeInterference(self, interf = 2, hei_interf = None, nhei_interf = None, offhei_interf = None):
1296 def removeInterference(self, interf = 2, hei_interf = None, nhei_interf = None, offhei_interf = None):
1298
1297
1299 jspectra = self.dataOut.data_spc
1298 jspectra = self.dataOut.data_spc
1300 jcspectra = self.dataOut.data_cspc
1299 jcspectra = self.dataOut.data_cspc
1301 jnoise = self.dataOut.getNoise()
1300 jnoise = self.dataOut.getNoise()
1302 num_incoh = self.dataOut.nIncohInt
1301 num_incoh = self.dataOut.nIncohInt
1303
1302
1304 num_channel = jspectra.shape[0]
1303 num_channel = jspectra.shape[0]
1305 num_prof = jspectra.shape[1]
1304 num_prof = jspectra.shape[1]
1306 num_hei = jspectra.shape[2]
1305 num_hei = jspectra.shape[2]
1307
1306
1308 # hei_interf
1307 # hei_interf
1309 if hei_interf is None:
1308 if hei_interf is None:
1310 count_hei = int(num_hei / 2)
1309 count_hei = int(num_hei / 2)
1311 hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
1310 hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
1312 hei_interf = numpy.asarray(hei_interf)[0]
1311 hei_interf = numpy.asarray(hei_interf)[0]
1313 # nhei_interf
1312 # nhei_interf
1314 if (nhei_interf == None):
1313 if (nhei_interf == None):
1315 nhei_interf = 5
1314 nhei_interf = 5
1316 if (nhei_interf < 1):
1315 if (nhei_interf < 1):
1317 nhei_interf = 1
1316 nhei_interf = 1
1318 if (nhei_interf > count_hei):
1317 if (nhei_interf > count_hei):
1319 nhei_interf = count_hei
1318 nhei_interf = count_hei
1320 if (offhei_interf == None):
1319 if (offhei_interf == None):
1321 offhei_interf = 0
1320 offhei_interf = 0
1322
1321
1323 ind_hei = list(range(num_hei))
1322 ind_hei = list(range(num_hei))
1324 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
1323 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
1325 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
1324 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
1326 mask_prof = numpy.asarray(list(range(num_prof)))
1325 mask_prof = numpy.asarray(list(range(num_prof)))
1327 num_mask_prof = mask_prof.size
1326 num_mask_prof = mask_prof.size
1328 comp_mask_prof = [0, num_prof / 2]
1327 comp_mask_prof = [0, num_prof / 2]
1329
1328
1330 # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
1329 # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
1331 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
1330 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
1332 jnoise = numpy.nan
1331 jnoise = numpy.nan
1333 noise_exist = jnoise[0] < numpy.Inf
1332 noise_exist = jnoise[0] < numpy.Inf
1334
1333
1335 # Subrutina de Remocion de la Interferencia
1334 # Subrutina de Remocion de la Interferencia
1336 for ich in range(num_channel):
1335 for ich in range(num_channel):
1337 # Se ordena los espectros segun su potencia (menor a mayor)
1336 # Se ordena los espectros segun su potencia (menor a mayor)
1338 power = jspectra[ich, mask_prof, :]
1337 power = jspectra[ich, mask_prof, :]
1339 power = power[:, hei_interf]
1338 power = power[:, hei_interf]
1340 power = power.sum(axis=0)
1339 power = power.sum(axis=0)
1341 psort = power.ravel().argsort()
1340 psort = power.ravel().argsort()
1342
1341
1343 # Se estima la interferencia promedio en los Espectros de Potencia empleando
1342 # Se estima la interferencia promedio en los Espectros de Potencia empleando
1344 junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
1343 junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
1345 offhei_interf, nhei_interf + offhei_interf))]]]
1344 offhei_interf, nhei_interf + offhei_interf))]]]
1346
1345
1347 if noise_exist:
1346 if noise_exist:
1348 # tmp_noise = jnoise[ich] / num_prof
1347 # tmp_noise = jnoise[ich] / num_prof
1349 tmp_noise = jnoise[ich]
1348 tmp_noise = jnoise[ich]
1350 junkspc_interf = junkspc_interf - tmp_noise
1349 junkspc_interf = junkspc_interf - tmp_noise
1351 #junkspc_interf[:,comp_mask_prof] = 0
1350 #junkspc_interf[:,comp_mask_prof] = 0
1352
1351
1353 jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
1352 jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
1354 jspc_interf = jspc_interf.transpose()
1353 jspc_interf = jspc_interf.transpose()
1355 # Calculando el espectro de interferencia promedio
1354 # Calculando el espectro de interferencia promedio
1356 noiseid = numpy.where(
1355 noiseid = numpy.where(
1357 jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
1356 jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
1358 noiseid = noiseid[0]
1357 noiseid = noiseid[0]
1359 cnoiseid = noiseid.size
1358 cnoiseid = noiseid.size
1360 interfid = numpy.where(
1359 interfid = numpy.where(
1361 jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
1360 jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
1362 interfid = interfid[0]
1361 interfid = interfid[0]
1363 cinterfid = interfid.size
1362 cinterfid = interfid.size
1364
1363
1365 if (cnoiseid > 0):
1364 if (cnoiseid > 0):
1366 jspc_interf[noiseid] = 0
1365 jspc_interf[noiseid] = 0
1367
1366
1368 # Expandiendo los perfiles a limpiar
1367 # Expandiendo los perfiles a limpiar
1369 if (cinterfid > 0):
1368 if (cinterfid > 0):
1370 new_interfid = (
1369 new_interfid = (
1371 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
1370 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
1372 new_interfid = numpy.asarray(new_interfid)
1371 new_interfid = numpy.asarray(new_interfid)
1373 new_interfid = {x for x in new_interfid}
1372 new_interfid = {x for x in new_interfid}
1374 new_interfid = numpy.array(list(new_interfid))
1373 new_interfid = numpy.array(list(new_interfid))
1375 new_cinterfid = new_interfid.size
1374 new_cinterfid = new_interfid.size
1376 else:
1375 else:
1377 new_cinterfid = 0
1376 new_cinterfid = 0
1378
1377
1379 for ip in range(new_cinterfid):
1378 for ip in range(new_cinterfid):
1380 ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
1379 ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
1381 jspc_interf[new_interfid[ip]
1380 jspc_interf[new_interfid[ip]
1382 ] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]]
1381 ] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]]
1383
1382
1384 jspectra[ich, :, ind_hei] = jspectra[ich, :,
1383 jspectra[ich, :, ind_hei] = jspectra[ich, :,
1385 ind_hei] - jspc_interf # Corregir indices
1384 ind_hei] - jspc_interf # Corregir indices
1386
1385
1387 # Removiendo la interferencia del punto de mayor interferencia
1386 # Removiendo la interferencia del punto de mayor interferencia
1388 ListAux = jspc_interf[mask_prof].tolist()
1387 ListAux = jspc_interf[mask_prof].tolist()
1389 maxid = ListAux.index(max(ListAux))
1388 maxid = ListAux.index(max(ListAux))
1390
1389
1391 if cinterfid > 0:
1390 if cinterfid > 0:
1392 for ip in range(cinterfid * (interf == 2) - 1):
1391 for ip in range(cinterfid * (interf == 2) - 1):
1393 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
1392 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
1394 (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
1393 (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
1395 cind = len(ind)
1394 cind = len(ind)
1396
1395
1397 if (cind > 0):
1396 if (cind > 0):
1398 jspectra[ich, interfid[ip], ind] = tmp_noise * \
1397 jspectra[ich, interfid[ip], ind] = tmp_noise * \
1399 (1 + (numpy.random.uniform(cind) - 0.5) /
1398 (1 + (numpy.random.uniform(cind) - 0.5) /
1400 numpy.sqrt(num_incoh))
1399 numpy.sqrt(num_incoh))
1401
1400
1402 ind = numpy.array([-2, -1, 1, 2])
1401 ind = numpy.array([-2, -1, 1, 2])
1403 xx = numpy.zeros([4, 4])
1402 xx = numpy.zeros([4, 4])
1404
1403
1405 for id1 in range(4):
1404 for id1 in range(4):
1406 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1405 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1407
1406
1408 xx_inv = numpy.linalg.inv(xx)
1407 xx_inv = numpy.linalg.inv(xx)
1409 xx = xx_inv[:, 0]
1408 xx = xx_inv[:, 0]
1410 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1409 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1411 yy = jspectra[ich, mask_prof[ind], :]
1410 yy = jspectra[ich, mask_prof[ind], :]
1412 jspectra[ich, mask_prof[maxid], :] = numpy.dot(
1411 jspectra[ich, mask_prof[maxid], :] = numpy.dot(
1413 yy.transpose(), xx)
1412 yy.transpose(), xx)
1414
1413
1415 indAux = (jspectra[ich, :, :] < tmp_noise *
1414 indAux = (jspectra[ich, :, :] < tmp_noise *
1416 (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
1415 (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
1417 jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
1416 jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
1418 (1 - 1 / numpy.sqrt(num_incoh))
1417 (1 - 1 / numpy.sqrt(num_incoh))
1419
1418
1420 # Remocion de Interferencia en el Cross Spectra
1419 # Remocion de Interferencia en el Cross Spectra
1421 if jcspectra is None:
1420 if jcspectra is None:
1422 return jspectra, jcspectra
1421 return jspectra, jcspectra
1423 num_pairs = int(jcspectra.size / (num_prof * num_hei))
1422 num_pairs = int(jcspectra.size / (num_prof * num_hei))
1424 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
1423 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
1425
1424
1426 for ip in range(num_pairs):
1425 for ip in range(num_pairs):
1427
1426
1428 #-------------------------------------------
1427 #-------------------------------------------
1429
1428
1430 cspower = numpy.abs(jcspectra[ip, mask_prof, :])
1429 cspower = numpy.abs(jcspectra[ip, mask_prof, :])
1431 cspower = cspower[:, hei_interf]
1430 cspower = cspower[:, hei_interf]
1432 cspower = cspower.sum(axis=0)
1431 cspower = cspower.sum(axis=0)
1433
1432
1434 cspsort = cspower.ravel().argsort()
1433 cspsort = cspower.ravel().argsort()
1435 junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
1434 junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
1436 offhei_interf, nhei_interf + offhei_interf))]]]
1435 offhei_interf, nhei_interf + offhei_interf))]]]
1437 junkcspc_interf = junkcspc_interf.transpose()
1436 junkcspc_interf = junkcspc_interf.transpose()
1438 jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
1437 jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
1439
1438
1440 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
1439 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
1441
1440
1442 median_real = int(numpy.median(numpy.real(
1441 median_real = int(numpy.median(numpy.real(
1443 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1442 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1444 median_imag = int(numpy.median(numpy.imag(
1443 median_imag = int(numpy.median(numpy.imag(
1445 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1444 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
1446 comp_mask_prof = [int(e) for e in comp_mask_prof]
1445 comp_mask_prof = [int(e) for e in comp_mask_prof]
1447 junkcspc_interf[comp_mask_prof, :] = numpy.complex(
1446 junkcspc_interf[comp_mask_prof, :] = numpy.complex(
1448 median_real, median_imag)
1447 median_real, median_imag)
1449
1448
1450 for iprof in range(num_prof):
1449 for iprof in range(num_prof):
1451 ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
1450 ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
1452 jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]]
1451 jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]]
1453
1452
1454 # Removiendo la Interferencia
1453 # Removiendo la Interferencia
1455 jcspectra[ip, :, ind_hei] = jcspectra[ip,
1454 jcspectra[ip, :, ind_hei] = jcspectra[ip,
1456 :, ind_hei] - jcspc_interf
1455 :, ind_hei] - jcspc_interf
1457
1456
1458 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
1457 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
1459 maxid = ListAux.index(max(ListAux))
1458 maxid = ListAux.index(max(ListAux))
1460
1459
1461 ind = numpy.array([-2, -1, 1, 2])
1460 ind = numpy.array([-2, -1, 1, 2])
1462 xx = numpy.zeros([4, 4])
1461 xx = numpy.zeros([4, 4])
1463
1462
1464 for id1 in range(4):
1463 for id1 in range(4):
1465 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1464 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
1466
1465
1467 xx_inv = numpy.linalg.inv(xx)
1466 xx_inv = numpy.linalg.inv(xx)
1468 xx = xx_inv[:, 0]
1467 xx = xx_inv[:, 0]
1469
1468
1470 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1469 ind = (ind + maxid + num_mask_prof) % num_mask_prof
1471 yy = jcspectra[ip, mask_prof[ind], :]
1470 yy = jcspectra[ip, mask_prof[ind], :]
1472 jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1471 jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
1473
1472
1474 # Guardar Resultados
1473 # Guardar Resultados
1475 self.dataOut.data_spc = jspectra
1474 self.dataOut.data_spc = jspectra
1476 self.dataOut.data_cspc = jcspectra
1475 self.dataOut.data_cspc = jcspectra
1477
1476
1478 return 1
1477 return 1
1479
1478
1480 def run(self, dataOut, interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None, mode=1):
1479 def run(self, dataOut, interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None, mode=1):
1481
1480
1482 self.dataOut = dataOut
1481 self.dataOut = dataOut
1483
1482
1484 if mode == 1:
1483 if mode == 1:
1485 self.removeInterference(interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None)
1484 self.removeInterference(interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None)
1486 elif mode == 2:
1485 elif mode == 2:
1487 self.removeInterference2()
1486 self.removeInterference2()
1488
1487
1489 return self.dataOut
1488 return self.dataOut
1490
1489
1491
1490
1492 class IncohInt(Operation):
1491 class IncohInt(Operation):
1493
1492
1494 __profIndex = 0
1493 __profIndex = 0
1495 __withOverapping = False
1494 __withOverapping = False
1496
1495
1497 __byTime = False
1496 __byTime = False
1498 __initime = None
1497 __initime = None
1499 __lastdatatime = None
1498 __lastdatatime = None
1500 __integrationtime = None
1499 __integrationtime = None
1501
1500
1502 __buffer_spc = None
1501 __buffer_spc = None
1503 __buffer_cspc = None
1502 __buffer_cspc = None
1504 __buffer_dc = None
1503 __buffer_dc = None
1505
1504
1506 __dataReady = False
1505 __dataReady = False
1507
1506
1508 __timeInterval = None
1507 __timeInterval = None
1509
1508
1510 n = None
1509 n = None
1511
1510
1512 def __init__(self):
1511 def __init__(self):
1513
1512
1514 Operation.__init__(self)
1513 Operation.__init__(self)
1515
1514
1516 def setup(self, n=None, timeInterval=None, overlapping=False):
1515 def setup(self, n=None, timeInterval=None, overlapping=False):
1517 """
1516 """
1518 Set the parameters of the integration class.
1517 Set the parameters of the integration class.
1519
1518
1520 Inputs:
1519 Inputs:
1521
1520
1522 n : Number of coherent integrations
1521 n : Number of coherent integrations
1523 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1522 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
1524 overlapping :
1523 overlapping :
1525
1524
1526 """
1525 """
1527
1526
1528 self.__initime = None
1527 self.__initime = None
1529 self.__lastdatatime = 0
1528 self.__lastdatatime = 0
1530
1529
1531 self.__buffer_spc = 0
1530 self.__buffer_spc = 0
1532 self.__buffer_cspc = 0
1531 self.__buffer_cspc = 0
1533 self.__buffer_dc = 0
1532 self.__buffer_dc = 0
1534
1533
1535 self.__profIndex = 0
1534 self.__profIndex = 0
1536 self.__dataReady = False
1535 self.__dataReady = False
1537 self.__byTime = False
1536 self.__byTime = False
1538
1537
1539 if n is None and timeInterval is None:
1538 if n is None and timeInterval is None:
1540 raise ValueError("n or timeInterval should be specified ...")
1539 raise ValueError("n or timeInterval should be specified ...")
1541
1540
1542 if n is not None:
1541 if n is not None:
1543 self.n = int(n)
1542 self.n = int(n)
1544 else:
1543 else:
1545
1544
1546 self.__integrationtime = int(timeInterval)
1545 self.__integrationtime = int(timeInterval)
1547 self.n = None
1546 self.n = None
1548 self.__byTime = True
1547 self.__byTime = True
1549
1548
1550 def putData(self, data_spc, data_cspc, data_dc):
1549 def putData(self, data_spc, data_cspc, data_dc):
1551 """
1550 """
1552 Add a profile to the __buffer_spc and increase in one the __profileIndex
1551 Add a profile to the __buffer_spc and increase in one the __profileIndex
1553
1552
1554 """
1553 """
1555
1554
1556 self.__buffer_spc += data_spc
1555 self.__buffer_spc += data_spc
1557
1556
1558 if data_cspc is None:
1557 if data_cspc is None:
1559 self.__buffer_cspc = None
1558 self.__buffer_cspc = None
1560 else:
1559 else:
1561 self.__buffer_cspc += data_cspc
1560 self.__buffer_cspc += data_cspc
1562
1561
1563 if data_dc is None:
1562 if data_dc is None:
1564 self.__buffer_dc = None
1563 self.__buffer_dc = None
1565 else:
1564 else:
1566 self.__buffer_dc += data_dc
1565 self.__buffer_dc += data_dc
1567
1566
1568 self.__profIndex += 1
1567 self.__profIndex += 1
1569
1568
1570 return
1569 return
1571
1570
1572 def pushData(self):
1571 def pushData(self):
1573 """
1572 """
1574 Return the sum of the last profiles and the profiles used in the sum.
1573 Return the sum of the last profiles and the profiles used in the sum.
1575
1574
1576 Affected:
1575 Affected:
1577
1576
1578 self.__profileIndex
1577 self.__profileIndex
1579
1578
1580 """
1579 """
1581
1580
1582 data_spc = self.__buffer_spc
1581 data_spc = self.__buffer_spc
1583 data_cspc = self.__buffer_cspc
1582 data_cspc = self.__buffer_cspc
1584 data_dc = self.__buffer_dc
1583 data_dc = self.__buffer_dc
1585 n = self.__profIndex
1584 n = self.__profIndex
1586
1585
1587 self.__buffer_spc = 0
1586 self.__buffer_spc = 0
1588 self.__buffer_cspc = 0
1587 self.__buffer_cspc = 0
1589 self.__buffer_dc = 0
1588 self.__buffer_dc = 0
1590 self.__profIndex = 0
1589 self.__profIndex = 0
1591
1590
1592 return data_spc, data_cspc, data_dc, n
1591 return data_spc, data_cspc, data_dc, n
1593
1592
1594 def byProfiles(self, *args):
1593 def byProfiles(self, *args):
1595
1594
1596 self.__dataReady = False
1595 self.__dataReady = False
1597 avgdata_spc = None
1596 avgdata_spc = None
1598 avgdata_cspc = None
1597 avgdata_cspc = None
1599 avgdata_dc = None
1598 avgdata_dc = None
1600
1599
1601 self.putData(*args)
1600 self.putData(*args)
1602
1601
1603 if self.__profIndex == self.n:
1602 if self.__profIndex == self.n:
1604
1603
1605 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1604 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1606 self.n = n
1605 self.n = n
1607 self.__dataReady = True
1606 self.__dataReady = True
1608
1607
1609 return avgdata_spc, avgdata_cspc, avgdata_dc
1608 return avgdata_spc, avgdata_cspc, avgdata_dc
1610
1609
1611 def byTime(self, datatime, *args):
1610 def byTime(self, datatime, *args):
1612
1611
1613 self.__dataReady = False
1612 self.__dataReady = False
1614 avgdata_spc = None
1613 avgdata_spc = None
1615 avgdata_cspc = None
1614 avgdata_cspc = None
1616 avgdata_dc = None
1615 avgdata_dc = None
1617
1616
1618 self.putData(*args)
1617 self.putData(*args)
1619
1618
1620 if (datatime - self.__initime) >= self.__integrationtime:
1619 if (datatime - self.__initime) >= self.__integrationtime:
1621 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1620 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
1622 self.n = n
1621 self.n = n
1623 self.__dataReady = True
1622 self.__dataReady = True
1624
1623
1625 return avgdata_spc, avgdata_cspc, avgdata_dc
1624 return avgdata_spc, avgdata_cspc, avgdata_dc
1626
1625
1627 def integrate(self, datatime, *args):
1626 def integrate(self, datatime, *args):
1628
1627
1629 if self.__profIndex == 0:
1628 if self.__profIndex == 0:
1630 self.__initime = datatime
1629 self.__initime = datatime
1631
1630
1632 if self.__byTime:
1631 if self.__byTime:
1633 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1632 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
1634 datatime, *args)
1633 datatime, *args)
1635 else:
1634 else:
1636 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1635 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
1637
1636
1638 if not self.__dataReady:
1637 if not self.__dataReady:
1639 return None, None, None, None
1638 return None, None, None, None
1640
1639
1641 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1640 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
1642
1641
1643 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
1642 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
1644 if n == 1:
1643 if n == 1:
1645 return dataOut
1644 return dataOut
1646
1645
1647 dataOut.flagNoData = True
1646 dataOut.flagNoData = True
1648
1647
1649 if not self.isConfig:
1648 if not self.isConfig:
1650 self.setup(n, timeInterval, overlapping)
1649 self.setup(n, timeInterval, overlapping)
1651 self.isConfig = True
1650 self.isConfig = True
1652
1651
1653 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
1652 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
1654 dataOut.data_spc,
1653 dataOut.data_spc,
1655 dataOut.data_cspc,
1654 dataOut.data_cspc,
1656 dataOut.data_dc)
1655 dataOut.data_dc)
1657
1656
1658 if self.__dataReady:
1657 if self.__dataReady:
1659
1658
1660 dataOut.data_spc = avgdata_spc
1659 dataOut.data_spc = avgdata_spc
1661 dataOut.data_cspc = avgdata_cspc
1660 dataOut.data_cspc = avgdata_cspc
1662 dataOut.data_dc = avgdata_dc
1661 dataOut.data_dc = avgdata_dc
1663 dataOut.nIncohInt *= self.n
1662 dataOut.nIncohInt *= self.n
1664 dataOut.utctime = avgdatatime
1663 dataOut.utctime = avgdatatime
1665 dataOut.flagNoData = False
1664 dataOut.flagNoData = False
1666
1665
1667 return dataOut
1666 return dataOut
1668
1667
1669 class dopplerFlip(Operation):
1668 class dopplerFlip(Operation):
1670
1669
1671 def run(self, dataOut):
1670 def run(self, dataOut):
1672 # arreglo 1: (num_chan, num_profiles, num_heights)
1671 # arreglo 1: (num_chan, num_profiles, num_heights)
1673 self.dataOut = dataOut
1672 self.dataOut = dataOut
1674 # JULIA-oblicua, indice 2
1673 # JULIA-oblicua, indice 2
1675 # arreglo 2: (num_profiles, num_heights)
1674 # arreglo 2: (num_profiles, num_heights)
1676 jspectra = self.dataOut.data_spc[2]
1675 jspectra = self.dataOut.data_spc[2]
1677 jspectra_tmp = numpy.zeros(jspectra.shape)
1676 jspectra_tmp = numpy.zeros(jspectra.shape)
1678 num_profiles = jspectra.shape[0]
1677 num_profiles = jspectra.shape[0]
1679 freq_dc = int(num_profiles / 2)
1678 freq_dc = int(num_profiles / 2)
1680 # Flip con for
1679 # Flip con for
1681 for j in range(num_profiles):
1680 for j in range(num_profiles):
1682 jspectra_tmp[num_profiles-j-1]= jspectra[j]
1681 jspectra_tmp[num_profiles-j-1]= jspectra[j]
1683 # Intercambio perfil de DC con perfil inmediato anterior
1682 # Intercambio perfil de DC con perfil inmediato anterior
1684 jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1]
1683 jspectra_tmp[freq_dc-1]= jspectra[freq_dc-1]
1685 jspectra_tmp[freq_dc]= jspectra[freq_dc]
1684 jspectra_tmp[freq_dc]= jspectra[freq_dc]
1686 # canal modificado es re-escrito en el arreglo de canales
1685 # canal modificado es re-escrito en el arreglo de canales
1687 self.dataOut.data_spc[2] = jspectra_tmp
1686 self.dataOut.data_spc[2] = jspectra_tmp
1688
1687
1689 return self.dataOut
1688 return self.dataOut
Show file before | Show file after | Hide comments
@@ -1,1638 +1,1758
1 import sys
1 import sys
2 import numpy,math
2 import numpy,math
3 from scipy import interpolate
3 from scipy import interpolate
4 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
4 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
5 from schainpy.model.data.jrodata import Voltage,hildebrand_sekhon
5 from schainpy.model.data.jrodata import Voltage,hildebrand_sekhon
6 from schainpy.utils import log
6 from schainpy.utils import log
7 from time import time
7 from time import time
8 import numpy
8 import numpy
9
9
10
10
11 class VoltageProc(ProcessingUnit):
11 class VoltageProc(ProcessingUnit):
12
12
13 def __init__(self):
13 def __init__(self):
14
14
15 ProcessingUnit.__init__(self)
15 ProcessingUnit.__init__(self)
16
16
17 self.dataOut = Voltage()
17 self.dataOut = Voltage()
18 self.flip = 1
18 self.flip = 1
19 self.setupReq = False
19 self.setupReq = False
20
20
21 def run(self):
21 def run(self):
22
22
23 if self.dataIn.type == 'AMISR':
23 if self.dataIn.type == 'AMISR':
24 self.__updateObjFromAmisrInput()
24 self.__updateObjFromAmisrInput()
25
25
26 if self.dataIn.type == 'Voltage':
26 if self.dataIn.type == 'Voltage':
27 self.dataOut.copy(self.dataIn)
27 self.dataOut.copy(self.dataIn)
28
28
29 def __updateObjFromAmisrInput(self):
29 def __updateObjFromAmisrInput(self):
30
30
31 self.dataOut.timeZone = self.dataIn.timeZone
31 self.dataOut.timeZone = self.dataIn.timeZone
32 self.dataOut.dstFlag = self.dataIn.dstFlag
32 self.dataOut.dstFlag = self.dataIn.dstFlag
33 self.dataOut.errorCount = self.dataIn.errorCount
33 self.dataOut.errorCount = self.dataIn.errorCount
34 self.dataOut.useLocalTime = self.dataIn.useLocalTime
34 self.dataOut.useLocalTime = self.dataIn.useLocalTime
35
35
36 self.dataOut.flagNoData = self.dataIn.flagNoData
36 self.dataOut.flagNoData = self.dataIn.flagNoData
37 self.dataOut.data = self.dataIn.data
37 self.dataOut.data = self.dataIn.data
38 self.dataOut.utctime = self.dataIn.utctime
38 self.dataOut.utctime = self.dataIn.utctime
39 self.dataOut.channelList = self.dataIn.channelList
39 self.dataOut.channelList = self.dataIn.channelList
40 #self.dataOut.timeInterval = self.dataIn.timeInterval
40 #self.dataOut.timeInterval = self.dataIn.timeInterval
41 self.dataOut.heightList = self.dataIn.heightList
41 self.dataOut.heightList = self.dataIn.heightList
42 self.dataOut.nProfiles = self.dataIn.nProfiles
42 self.dataOut.nProfiles = self.dataIn.nProfiles
43
43
44 self.dataOut.nCohInt = self.dataIn.nCohInt
44 self.dataOut.nCohInt = self.dataIn.nCohInt
45 self.dataOut.ippSeconds = self.dataIn.ippSeconds
45 self.dataOut.ippSeconds = self.dataIn.ippSeconds
46 self.dataOut.frequency = self.dataIn.frequency
46 self.dataOut.frequency = self.dataIn.frequency
47
47
48 self.dataOut.azimuth = self.dataIn.azimuth
48 self.dataOut.azimuth = self.dataIn.azimuth
49 self.dataOut.zenith = self.dataIn.zenith
49 self.dataOut.zenith = self.dataIn.zenith
50
50
51 self.dataOut.beam.codeList = self.dataIn.beam.codeList
51 self.dataOut.beam.codeList = self.dataIn.beam.codeList
52 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
52 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
53 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
53 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
54
54
55
55
56 class selectChannels(Operation):
56 class selectChannels(Operation):
57
57
58 def run(self, dataOut, channelList=None):
58 def run(self, dataOut, channelList=None):
59 self.channelList = channelList
59 self.channelList = channelList
60 if self.channelList == None:
60 if self.channelList == None:
61 print("Missing channelList")
61 print("Missing channelList")
62 return dataOut
62 return dataOut
63 channelIndexList = []
63 channelIndexList = []
64
64
65 if type(dataOut.channelList) is not list: #leer array desde HDF5
65 if type(dataOut.channelList) is not list: #leer array desde HDF5
66 try:
66 try:
67 dataOut.channelList = dataOut.channelList.tolist()
67 dataOut.channelList = dataOut.channelList.tolist()
68 except Exception as e:
68 except Exception as e:
69 print("Select Channels: ",e)
69 print("Select Channels: ",e)
70 for channel in self.channelList:
70 for channel in self.channelList:
71 if channel not in dataOut.channelList:
71 if channel not in dataOut.channelList:
72 raise ValueError("Channel %d is not in %s" %(channel, str(dataOut.channelList)))
72 raise ValueError("Channel %d is not in %s" %(channel, str(dataOut.channelList)))
73
73
74 index = dataOut.channelList.index(channel)
74 index = dataOut.channelList.index(channel)
75 channelIndexList.append(index)
75 channelIndexList.append(index)
76 dataOut = self.selectChannelsByIndex(dataOut,channelIndexList)
76 dataOut = self.selectChannelsByIndex(dataOut,channelIndexList)
77 return dataOut
77 return dataOut
78
78
79 def selectChannelsByIndex(self, dataOut, channelIndexList):
79 def selectChannelsByIndex(self, dataOut, channelIndexList):
80 """
80 """
81 Selecciona un bloque de datos en base a canales segun el channelIndexList
81 Selecciona un bloque de datos en base a canales segun el channelIndexList
82
82
83 Input:
83 Input:
84 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
84 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
85
85
86 Affected:
86 Affected:
87 dataOut.data
87 dataOut.data
88 dataOut.channelIndexList
88 dataOut.channelIndexList
89 dataOut.nChannels
89 dataOut.nChannels
90 dataOut.m_ProcessingHeader.totalSpectra
90 dataOut.m_ProcessingHeader.totalSpectra
91 dataOut.systemHeaderObj.numChannels
91 dataOut.systemHeaderObj.numChannels
92 dataOut.m_ProcessingHeader.blockSize
92 dataOut.m_ProcessingHeader.blockSize
93
93
94 Return:
94 Return:
95 None
95 None
96 """
96 """
97 #print("selectChannelsByIndex")
97 #print("selectChannelsByIndex")
98 # for channelIndex in channelIndexList:
98 # for channelIndex in channelIndexList:
99 # if channelIndex not in dataOut.channelIndexList:
99 # if channelIndex not in dataOut.channelIndexList:
100 # raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
100 # raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
101
101
102 if dataOut.type == 'Voltage':
102 if dataOut.type == 'Voltage':
103 if dataOut.flagDataAsBlock:
103 if dataOut.flagDataAsBlock:
104 """
104 """
105 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
105 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
106 """
106 """
107 data = dataOut.data[channelIndexList,:,:]
107 data = dataOut.data[channelIndexList,:,:]
108 else:
108 else:
109 data = dataOut.data[channelIndexList,:]
109 data = dataOut.data[channelIndexList,:]
110
110
111 dataOut.data = data
111 dataOut.data = data
112 # dataOut.channelList = [dataOut.channelList[i] for i in channelIndexList]
112 # dataOut.channelList = [dataOut.channelList[i] for i in channelIndexList]
113 dataOut.channelList = range(len(channelIndexList))
113 dataOut.channelList = range(len(channelIndexList))
114
114
115 elif dataOut.type == 'Spectra':
115 elif dataOut.type == 'Spectra':
116 if hasattr(dataOut, 'data_spc'):
116 if hasattr(dataOut, 'data_spc'):
117 if dataOut.data_spc is None:
117 if dataOut.data_spc is None:
118 raise ValueError("data_spc is None")
118 raise ValueError("data_spc is None")
119 return dataOut
119 return dataOut
120 else:
120 else:
121 data_spc = dataOut.data_spc[channelIndexList, :]
121 data_spc = dataOut.data_spc[channelIndexList, :]
122 dataOut.data_spc = data_spc
122 dataOut.data_spc = data_spc
123
123
124 # if hasattr(dataOut, 'data_dc') :# and
124 # if hasattr(dataOut, 'data_dc') :# and
125 # if dataOut.data_dc is None:
125 # if dataOut.data_dc is None:
126 # raise ValueError("data_dc is None")
126 # raise ValueError("data_dc is None")
127 # return dataOut
127 # return dataOut
128 # else:
128 # else:
129 # data_dc = dataOut.data_dc[channelIndexList, :]
129 # data_dc = dataOut.data_dc[channelIndexList, :]
130 # dataOut.data_dc = data_dc
130 # dataOut.data_dc = data_dc
131 # dataOut.channelList = [dataOut.channelList[i] for i in channelIndexList]
131 # dataOut.channelList = [dataOut.channelList[i] for i in channelIndexList]
132 dataOut.channelList = channelIndexList
132 dataOut.channelList = channelIndexList
133 dataOut = self.__selectPairsByChannel(dataOut,channelIndexList)
133 dataOut = self.__selectPairsByChannel(dataOut,channelIndexList)
134
134
135 return dataOut
135 return dataOut
136
136
137 def __selectPairsByChannel(self, dataOut, channelList=None):
137 def __selectPairsByChannel(self, dataOut, channelList=None):
138 #print("__selectPairsByChannel")
138 #print("__selectPairsByChannel")
139 if channelList == None:
139 if channelList == None:
140 return
140 return
141
141
142 pairsIndexListSelected = []
142 pairsIndexListSelected = []
143 for pairIndex in dataOut.pairsIndexList:
143 for pairIndex in dataOut.pairsIndexList:
144 # First pair
144 # First pair
145 if dataOut.pairsList[pairIndex][0] not in channelList:
145 if dataOut.pairsList[pairIndex][0] not in channelList:
146 continue
146 continue
147 # Second pair
147 # Second pair
148 if dataOut.pairsList[pairIndex][1] not in channelList:
148 if dataOut.pairsList[pairIndex][1] not in channelList:
149 continue
149 continue
150
150
151 pairsIndexListSelected.append(pairIndex)
151 pairsIndexListSelected.append(pairIndex)
152 if not pairsIndexListSelected:
152 if not pairsIndexListSelected:
153 dataOut.data_cspc = None
153 dataOut.data_cspc = None
154 dataOut.pairsList = []
154 dataOut.pairsList = []
155 return
155 return
156
156
157 dataOut.data_cspc = dataOut.data_cspc[pairsIndexListSelected]
157 dataOut.data_cspc = dataOut.data_cspc[pairsIndexListSelected]
158 dataOut.pairsList = [dataOut.pairsList[i]
158 dataOut.pairsList = [dataOut.pairsList[i]
159 for i in pairsIndexListSelected]
159 for i in pairsIndexListSelected]
160
160
161 return dataOut
161 return dataOut
162
162
163 class selectHeights(Operation):
163 class selectHeights(Operation):
164
164
165 def run(self, dataOut, minHei=None, maxHei=None, minIndex=None, maxIndex=None):
165 def run(self, dataOut, minHei=None, maxHei=None, minIndex=None, maxIndex=None):
166 """
166 """
167 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
167 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
168 minHei <= height <= maxHei
168 minHei <= height <= maxHei
169
169
170 Input:
170 Input:
171 minHei : valor minimo de altura a considerar
171 minHei : valor minimo de altura a considerar
172 maxHei : valor maximo de altura a considerar
172 maxHei : valor maximo de altura a considerar
173
173
174 Affected:
174 Affected:
175 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
175 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
176
176
177 Return:
177 Return:
178 1 si el metodo se ejecuto con exito caso contrario devuelve 0
178 1 si el metodo se ejecuto con exito caso contrario devuelve 0
179 """
179 """
180
180
181 dataOut = dataOut
181 self.dataOut = dataOut
182
182
183 if minHei and maxHei:
183 if minHei and maxHei:
184
184
185 if (minHei < dataOut.heightList[0]):
185 if (minHei < dataOut.heightList[0]):
186 minHei = dataOut.heightList[0]
186 minHei = dataOut.heightList[0]
187
187
188 if (maxHei > dataOut.heightList[-1]):
188 if (maxHei > dataOut.heightList[-1]):
189 maxHei = dataOut.heightList[-1]
189 maxHei = dataOut.heightList[-1]
190
190
191 minIndex = 0
191 minIndex = 0
192 maxIndex = 0
192 maxIndex = 0
193 heights = dataOut.heightList
193 heights = dataOut.heightList
194
194
195 inda = numpy.where(heights >= minHei)
195 inda = numpy.where(heights >= minHei)
196 indb = numpy.where(heights <= maxHei)
196 indb = numpy.where(heights <= maxHei)
197
197
198 try:
198 try:
199 minIndex = inda[0][0]
199 minIndex = inda[0][0]
200 except:
200 except:
201 minIndex = 0
201 minIndex = 0
202
202
203 try:
203 try:
204 maxIndex = indb[0][-1]
204 maxIndex = indb[0][-1]
205 except:
205 except:
206 maxIndex = len(heights)
206 maxIndex = len(heights)
207
207
208 self.selectHeightsByIndex(minIndex, maxIndex)
208 self.selectHeightsByIndex(minIndex, maxIndex)
209
209
210 return self.dataOut
210 return dataOut
211
211
212 def selectHeightsByIndex(self, minIndex, maxIndex):
212 def selectHeightsByIndex(self, minIndex, maxIndex):
213 """
213 """
214 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
214 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
215 minIndex <= index <= maxIndex
215 minIndex <= index <= maxIndex
216
216
217 Input:
217 Input:
218 minIndex : valor de indice minimo de altura a considerar
218 minIndex : valor de indice minimo de altura a considerar
219 maxIndex : valor de indice maximo de altura a considerar
219 maxIndex : valor de indice maximo de altura a considerar
220
220
221 Affected:
221 Affected:
222 self.dataOut.data
222 self.dataOut.data
223 self.dataOut.heightList
223 self.dataOut.heightList
224
224
225 Return:
225 Return:
226 1 si el metodo se ejecuto con exito caso contrario devuelve 0
226 1 si el metodo se ejecuto con exito caso contrario devuelve 0
227 """
227 """
228
228
229 if self.dataOut.type == 'Voltage':
229 if self.dataOut.type == 'Voltage':
230 if (minIndex < 0) or (minIndex > maxIndex):
230 if (minIndex < 0) or (minIndex > maxIndex):
231 raise ValueError("Height index range (%d,%d) is not valid" % (minIndex, maxIndex))
231 raise ValueError("Height index range (%d,%d) is not valid" % (minIndex, maxIndex))
232
232
233 if (maxIndex >= self.dataOut.nHeights):
233 if (maxIndex >= self.dataOut.nHeights):
234 maxIndex = self.dataOut.nHeights
234 maxIndex = self.dataOut.nHeights
235
235
236 #voltage
236 #voltage
237 if self.dataOut.flagDataAsBlock:
237 if self.dataOut.flagDataAsBlock:
238 """
238 """
239 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
239 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
240 """
240 """
241 data = self.dataOut.data[:,:, minIndex:maxIndex]
241 data = self.dataOut.data[:,:, minIndex:maxIndex]
242 else:
242 else:
243 data = self.dataOut.data[:, minIndex:maxIndex]
243 data = self.dataOut.data[:, minIndex:maxIndex]
244
244
245 # firstHeight = self.dataOut.heightList[minIndex]
245 # firstHeight = self.dataOut.heightList[minIndex]
246
246
247 self.dataOut.data = data
247 self.dataOut.data = data
248 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
248 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
249
249
250 if self.dataOut.nHeights <= 1:
250 if self.dataOut.nHeights <= 1:
251 raise ValueError("selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights))
251 raise ValueError("selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights))
252 elif self.dataOut.type == 'Spectra':
252 elif self.dataOut.type == 'Spectra':
253 if (minIndex < 0) or (minIndex > maxIndex):
253 if (minIndex < 0) or (minIndex > maxIndex):
254 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (
254 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (
255 minIndex, maxIndex))
255 minIndex, maxIndex))
256
256
257 if (maxIndex >= self.dataOut.nHeights):
257 if (maxIndex >= self.dataOut.nHeights):
258 maxIndex = self.dataOut.nHeights - 1
258 maxIndex = self.dataOut.nHeights - 1
259
259
260 # Spectra
260 # Spectra
261 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
261 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
262
262
263 data_cspc = None
263 data_cspc = None
264 if self.dataOut.data_cspc is not None:
264 if self.dataOut.data_cspc is not None:
265 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
265 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
266
266
267 data_dc = None
267 data_dc = None
268 if self.dataOut.data_dc is not None:
268 if self.dataOut.data_dc is not None:
269 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
269 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
270
270
271 self.dataOut.data_spc = data_spc
271 self.dataOut.data_spc = data_spc
272 self.dataOut.data_cspc = data_cspc
272 self.dataOut.data_cspc = data_cspc
273 self.dataOut.data_dc = data_dc
273 self.dataOut.data_dc = data_dc
274
274
275 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
275 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
276
276
277 return 1
277 return 1
278
278
279
279
280 class filterByHeights(Operation):
280 class filterByHeights(Operation):
281
281
282 def run(self, dataOut, window):
282 def run(self, dataOut, window):
283
283
284 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
284 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
285
285
286 if window == None:
286 if window == None:
287 window = (dataOut.radarControllerHeaderObj.txA/dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
287 window = (dataOut.radarControllerHeaderObj.txA/dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
288
288
289 newdelta = deltaHeight * window
289 newdelta = deltaHeight * window
290 r = dataOut.nHeights % window
290 r = dataOut.nHeights % window
291 newheights = (dataOut.nHeights-r)/window
291 newheights = (dataOut.nHeights-r)/window
292
292
293 if newheights <= 1:
293 if newheights <= 1:
294 raise ValueError("filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(dataOut.nHeights, window))
294 raise ValueError("filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(dataOut.nHeights, window))
295
295
296 if dataOut.flagDataAsBlock:
296 if dataOut.flagDataAsBlock:
297 """
297 """
298 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
298 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
299 """
299 """
300 buffer = dataOut.data[:, :, 0:int(dataOut.nHeights-r)]
300 buffer = dataOut.data[:, :, 0:int(dataOut.nHeights-r)]
301 buffer = buffer.reshape(dataOut.nChannels, dataOut.nProfiles, int(dataOut.nHeights/window), window)
301 buffer = buffer.reshape(dataOut.nChannels, dataOut.nProfiles, int(dataOut.nHeights/window), window)
302 buffer = numpy.sum(buffer,3)
302 buffer = numpy.sum(buffer,3)
303
303
304 else:
304 else:
305 buffer = dataOut.data[:,0:int(dataOut.nHeights-r)]
305 buffer = dataOut.data[:,0:int(dataOut.nHeights-r)]
306 buffer = buffer.reshape(dataOut.nChannels,int(dataOut.nHeights/window),int(window))
306 buffer = buffer.reshape(dataOut.nChannels,int(dataOut.nHeights/window),int(window))
307 buffer = numpy.sum(buffer,2)
307 buffer = numpy.sum(buffer,2)
308
308
309 dataOut.data = buffer
309 dataOut.data = buffer
310 dataOut.heightList = dataOut.heightList[0] + numpy.arange( newheights )*newdelta
310 dataOut.heightList = dataOut.heightList[0] + numpy.arange( newheights )*newdelta
311 dataOut.windowOfFilter = window
311 dataOut.windowOfFilter = window
312
312
313 return dataOut
313 return dataOut
314
314
315
315
316 class setH0(Operation):
316 class setH0(Operation):
317
317
318 def run(self, dataOut, h0, deltaHeight = None):
318 def run(self, dataOut, h0, deltaHeight = None):
319
319
320 if not deltaHeight:
320 if not deltaHeight:
321 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
321 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
322
322
323 nHeights = dataOut.nHeights
323 nHeights = dataOut.nHeights
324
324
325 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
325 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
326
326
327 dataOut.heightList = newHeiRange
327 dataOut.heightList = newHeiRange
328
328
329 return dataOut
329 return dataOut
330
330
331
331
332 class deFlip(Operation):
332 class deFlip(Operation):
333
333
334 def run(self, dataOut, channelList = []):
334 def run(self, dataOut, channelList = []):
335
335
336 data = dataOut.data.copy()
336 data = dataOut.data.copy()
337
337
338 if dataOut.flagDataAsBlock:
338 if dataOut.flagDataAsBlock:
339 flip = self.flip
339 flip = self.flip
340 profileList = list(range(dataOut.nProfiles))
340 profileList = list(range(dataOut.nProfiles))
341
341
342 if not channelList:
342 if not channelList:
343 for thisProfile in profileList:
343 for thisProfile in profileList:
344 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
344 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
345 flip *= -1.0
345 flip *= -1.0
346 else:
346 else:
347 for thisChannel in channelList:
347 for thisChannel in channelList:
348 if thisChannel not in dataOut.channelList:
348 if thisChannel not in dataOut.channelList:
349 continue
349 continue
350
350
351 for thisProfile in profileList:
351 for thisProfile in profileList:
352 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
352 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
353 flip *= -1.0
353 flip *= -1.0
354
354
355 self.flip = flip
355 self.flip = flip
356
356
357 else:
357 else:
358 if not channelList:
358 if not channelList:
359 data[:,:] = data[:,:]*self.flip
359 data[:,:] = data[:,:]*self.flip
360 else:
360 else:
361 for thisChannel in channelList:
361 for thisChannel in channelList:
362 if thisChannel not in dataOut.channelList:
362 if thisChannel not in dataOut.channelList:
363 continue
363 continue
364
364
365 data[thisChannel,:] = data[thisChannel,:]*self.flip
365 data[thisChannel,:] = data[thisChannel,:]*self.flip
366
366
367 self.flip *= -1.
367 self.flip *= -1.
368
368
369 dataOut.data = data
369 dataOut.data = data
370
370
371 return dataOut
371 return dataOut
372
372
373
373
374 class setAttribute(Operation):
374 class setAttribute(Operation):
375 '''
375 '''
376 Set an arbitrary attribute(s) to dataOut
376 Set an arbitrary attribute(s) to dataOut
377 '''
377 '''
378
378
379 def __init__(self):
379 def __init__(self):
380
380
381 Operation.__init__(self)
381 Operation.__init__(self)
382 self._ready = False
382 self._ready = False
383
383
384 def run(self, dataOut, **kwargs):
384 def run(self, dataOut, **kwargs):
385
385
386 for key, value in kwargs.items():
386 for key, value in kwargs.items():
387 setattr(dataOut, key, value)
387 setattr(dataOut, key, value)
388
388
389 return dataOut
389 return dataOut
390
390
391
391
392 @MPDecorator
392 @MPDecorator
393 class printAttribute(Operation):
393 class printAttribute(Operation):
394 '''
394 '''
395 Print an arbitrary attribute of dataOut
395 Print an arbitrary attribute of dataOut
396 '''
396 '''
397
397
398 def __init__(self):
398 def __init__(self):
399
399
400 Operation.__init__(self)
400 Operation.__init__(self)
401
401
402 def run(self, dataOut, attributes):
402 def run(self, dataOut, attributes):
403
403
404 if isinstance(attributes, str):
404 if isinstance(attributes, str):
405 attributes = [attributes]
405 attributes = [attributes]
406 for attr in attributes:
406 for attr in attributes:
407 if hasattr(dataOut, attr):
407 if hasattr(dataOut, attr):
408 log.log(getattr(dataOut, attr), attr)
408 log.log(getattr(dataOut, attr), attr)
409
409
410
410
411 class interpolateHeights(Operation):
411 class interpolateHeights(Operation):
412
412
413 def run(self, dataOut, topLim, botLim):
413 def run(self, dataOut, topLim, botLim):
414 #69 al 72 para julia
414 #69 al 72 para julia
415 #82-84 para meteoros
415 #82-84 para meteoros
416 if len(numpy.shape(dataOut.data))==2:
416 if len(numpy.shape(dataOut.data))==2:
417 sampInterp = (dataOut.data[:,botLim-1] + dataOut.data[:,topLim+1])/2
417 sampInterp = (dataOut.data[:,botLim-1] + dataOut.data[:,topLim+1])/2
418 sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
418 sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
419 #dataOut.data[:,botLim:limSup+1] = sampInterp
419 #dataOut.data[:,botLim:limSup+1] = sampInterp
420 dataOut.data[:,botLim:topLim+1] = sampInterp
420 dataOut.data[:,botLim:topLim+1] = sampInterp
421 else:
421 else:
422 nHeights = dataOut.data.shape[2]
422 nHeights = dataOut.data.shape[2]
423 x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
423 x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
424 y = dataOut.data[:,:,list(range(botLim))+list(range(topLim+1,nHeights))]
424 y = dataOut.data[:,:,list(range(botLim))+list(range(topLim+1,nHeights))]
425 f = interpolate.interp1d(x, y, axis = 2)
425 f = interpolate.interp1d(x, y, axis = 2)
426 xnew = numpy.arange(botLim,topLim+1)
426 xnew = numpy.arange(botLim,topLim+1)
427 ynew = f(xnew)
427 ynew = f(xnew)
428 dataOut.data[:,:,botLim:topLim+1] = ynew
428 dataOut.data[:,:,botLim:topLim+1] = ynew
429
429
430 return dataOut
430 return dataOut
431
431
432
432
433 class CohInt(Operation):
433 class CohInt(Operation):
434
434
435 isConfig = False
435 isConfig = False
436 __profIndex = 0
436 __profIndex = 0
437 __byTime = False
437 __byTime = False
438 __initime = None
438 __initime = None
439 __lastdatatime = None
439 __lastdatatime = None
440 __integrationtime = None
440 __integrationtime = None
441 __buffer = None
441 __buffer = None
442 __bufferStride = []
442 __bufferStride = []
443 __dataReady = False
443 __dataReady = False
444 __profIndexStride = 0
444 __profIndexStride = 0
445 __dataToPutStride = False
445 __dataToPutStride = False
446 n = None
446 n = None
447
447
448 def __init__(self, **kwargs):
448 def __init__(self, **kwargs):
449
449
450 Operation.__init__(self, **kwargs)
450 Operation.__init__(self, **kwargs)
451
451
452 def setup(self, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False):
452 def setup(self, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False):
453 """
453 """
454 Set the parameters of the integration class.
454 Set the parameters of the integration class.
455
455
456 Inputs:
456 Inputs:
457
457
458 n : Number of coherent integrations
458 n : Number of coherent integrations
459 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
459 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
460 overlapping :
460 overlapping :
461 """
461 """
462
462
463 self.__initime = None
463 self.__initime = None
464 self.__lastdatatime = 0
464 self.__lastdatatime = 0
465 self.__buffer = None
465 self.__buffer = None
466 self.__dataReady = False
466 self.__dataReady = False
467 self.byblock = byblock
467 self.byblock = byblock
468 self.stride = stride
468 self.stride = stride
469
469
470 if n == None and timeInterval == None:
470 if n == None and timeInterval == None:
471 raise ValueError("n or timeInterval should be specified ...")
471 raise ValueError("n or timeInterval should be specified ...")
472
472
473 if n != None:
473 if n != None:
474 self.n = n
474 self.n = n
475 self.__byTime = False
475 self.__byTime = False
476 else:
476 else:
477 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
477 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
478 self.n = 9999
478 self.n = 9999
479 self.__byTime = True
479 self.__byTime = True
480
480
481 if overlapping:
481 if overlapping:
482 self.__withOverlapping = True
482 self.__withOverlapping = True
483 self.__buffer = None
483 self.__buffer = None
484 else:
484 else:
485 self.__withOverlapping = False
485 self.__withOverlapping = False
486 self.__buffer = 0
486 self.__buffer = 0
487
487
488 self.__profIndex = 0
488 self.__profIndex = 0
489
489
490 def putData(self, data):
490 def putData(self, data):
491
491
492 """
492 """
493 Add a profile to the __buffer and increase in one the __profileIndex
493 Add a profile to the __buffer and increase in one the __profileIndex
494
494
495 """
495 """
496
496
497 if not self.__withOverlapping:
497 if not self.__withOverlapping:
498 self.__buffer += data.copy()
498 self.__buffer += data.copy()
499 self.__profIndex += 1
499 self.__profIndex += 1
500 return
500 return
501
501
502 #Overlapping data
502 #Overlapping data
503 nChannels, nHeis = data.shape
503 nChannels, nHeis = data.shape
504 data = numpy.reshape(data, (1, nChannels, nHeis))
504 data = numpy.reshape(data, (1, nChannels, nHeis))
505
505
506 #If the buffer is empty then it takes the data value
506 #If the buffer is empty then it takes the data value
507 if self.__buffer is None:
507 if self.__buffer is None:
508 self.__buffer = data
508 self.__buffer = data
509 self.__profIndex += 1
509 self.__profIndex += 1
510 return
510 return
511
511
512 #If the buffer length is lower than n then stakcing the data value
512 #If the buffer length is lower than n then stakcing the data value
513 if self.__profIndex < self.n:
513 if self.__profIndex < self.n:
514 self.__buffer = numpy.vstack((self.__buffer, data))
514 self.__buffer = numpy.vstack((self.__buffer, data))
515 self.__profIndex += 1
515 self.__profIndex += 1
516 return
516 return
517
517
518 #If the buffer length is equal to n then replacing the last buffer value with the data value
518 #If the buffer length is equal to n then replacing the last buffer value with the data value
519 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
519 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
520 self.__buffer[self.n-1] = data
520 self.__buffer[self.n-1] = data
521 self.__profIndex = self.n
521 self.__profIndex = self.n
522 return
522 return
523
523
524
524
525 def pushData(self):
525 def pushData(self):
526 """
526 """
527 Return the sum of the last profiles and the profiles used in the sum.
527 Return the sum of the last profiles and the profiles used in the sum.
528
528
529 Affected:
529 Affected:
530
530
531 self.__profileIndex
531 self.__profileIndex
532
532
533 """
533 """
534
534
535 if not self.__withOverlapping:
535 if not self.__withOverlapping:
536 data = self.__buffer
536 data = self.__buffer
537 n = self.__profIndex
537 n = self.__profIndex
538
538
539 self.__buffer = 0
539 self.__buffer = 0
540 self.__profIndex = 0
540 self.__profIndex = 0
541
541
542 return data, n
542 return data, n
543
543
544 #Integration with Overlapping
544 #Integration with Overlapping
545 data = numpy.sum(self.__buffer, axis=0)
545 data = numpy.sum(self.__buffer, axis=0)
546 # print data
546 # print data
547 # raise
547 # raise
548 n = self.__profIndex
548 n = self.__profIndex
549
549
550 return data, n
550 return data, n
551
551
552 def byProfiles(self, data):
552 def byProfiles(self, data):
553
553
554 self.__dataReady = False
554 self.__dataReady = False
555 avgdata = None
555 avgdata = None
556 # n = None
556 # n = None
557 # print data
557 # print data
558 # raise
558 # raise
559 self.putData(data)
559 self.putData(data)
560
560
561 if self.__profIndex == self.n:
561 if self.__profIndex == self.n:
562 avgdata, n = self.pushData()
562 avgdata, n = self.pushData()
563 self.__dataReady = True
563 self.__dataReady = True
564
564
565 return avgdata
565 return avgdata
566
566
567 def byTime(self, data, datatime):
567 def byTime(self, data, datatime):
568
568
569 self.__dataReady = False
569 self.__dataReady = False
570 avgdata = None
570 avgdata = None
571 n = None
571 n = None
572
572
573 self.putData(data)
573 self.putData(data)
574
574
575 if (datatime - self.__initime) >= self.__integrationtime:
575 if (datatime - self.__initime) >= self.__integrationtime:
576 avgdata, n = self.pushData()
576 avgdata, n = self.pushData()
577 self.n = n
577 self.n = n
578 self.__dataReady = True
578 self.__dataReady = True
579
579
580 return avgdata
580 return avgdata
581
581
582 def integrateByStride(self, data, datatime):
582 def integrateByStride(self, data, datatime):
583 # print data
583 # print data
584 if self.__profIndex == 0:
584 if self.__profIndex == 0:
585 self.__buffer = [[data.copy(), datatime]]
585 self.__buffer = [[data.copy(), datatime]]
586 else:
586 else:
587 self.__buffer.append([data.copy(),datatime])
587 self.__buffer.append([data.copy(),datatime])
588 self.__profIndex += 1
588 self.__profIndex += 1
589 self.__dataReady = False
589 self.__dataReady = False
590
590
591 if self.__profIndex == self.n * self.stride :
591 if self.__profIndex == self.n * self.stride :
592 self.__dataToPutStride = True
592 self.__dataToPutStride = True
593 self.__profIndexStride = 0
593 self.__profIndexStride = 0
594 self.__profIndex = 0
594 self.__profIndex = 0
595 self.__bufferStride = []
595 self.__bufferStride = []
596 for i in range(self.stride):
596 for i in range(self.stride):
597 current = self.__buffer[i::self.stride]
597 current = self.__buffer[i::self.stride]
598 data = numpy.sum([t[0] for t in current], axis=0)
598 data = numpy.sum([t[0] for t in current], axis=0)
599 avgdatatime = numpy.average([t[1] for t in current])
599 avgdatatime = numpy.average([t[1] for t in current])
600 # print data
600 # print data
601 self.__bufferStride.append((data, avgdatatime))
601 self.__bufferStride.append((data, avgdatatime))
602
602
603 if self.__dataToPutStride:
603 if self.__dataToPutStride:
604 self.__dataReady = True
604 self.__dataReady = True
605 self.__profIndexStride += 1
605 self.__profIndexStride += 1
606 if self.__profIndexStride == self.stride:
606 if self.__profIndexStride == self.stride:
607 self.__dataToPutStride = False
607 self.__dataToPutStride = False
608 # print self.__bufferStride[self.__profIndexStride - 1]
608 # print self.__bufferStride[self.__profIndexStride - 1]
609 # raise
609 # raise
610 return self.__bufferStride[self.__profIndexStride - 1]
610 return self.__bufferStride[self.__profIndexStride - 1]
611
611
612
612
613 return None, None
613 return None, None
614
614
615 def integrate(self, data, datatime=None):
615 def integrate(self, data, datatime=None):
616
616
617 if self.__initime == None:
617 if self.__initime == None:
618 self.__initime = datatime
618 self.__initime = datatime
619
619
620 if self.__byTime:
620 if self.__byTime:
621 avgdata = self.byTime(data, datatime)
621 avgdata = self.byTime(data, datatime)
622 else:
622 else:
623 avgdata = self.byProfiles(data)
623 avgdata = self.byProfiles(data)
624
624
625
625
626 self.__lastdatatime = datatime
626 self.__lastdatatime = datatime
627
627
628 if avgdata is None:
628 if avgdata is None:
629 return None, None
629 return None, None
630
630
631 avgdatatime = self.__initime
631 avgdatatime = self.__initime
632
632
633 deltatime = datatime - self.__lastdatatime
633 deltatime = datatime - self.__lastdatatime
634
634
635 if not self.__withOverlapping:
635 if not self.__withOverlapping:
636 self.__initime = datatime
636 self.__initime = datatime
637 else:
637 else:
638 self.__initime += deltatime
638 self.__initime += deltatime
639
639
640 return avgdata, avgdatatime
640 return avgdata, avgdatatime
641
641
642 def integrateByBlock(self, dataOut):
642 def integrateByBlock(self, dataOut):
643
643
644 times = int(dataOut.data.shape[1]/self.n)
644 times = int(dataOut.data.shape[1]/self.n)
645 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
645 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
646
646
647 id_min = 0
647 id_min = 0
648 id_max = self.n
648 id_max = self.n
649
649
650 for i in range(times):
650 for i in range(times):
651 junk = dataOut.data[:,id_min:id_max,:]
651 junk = dataOut.data[:,id_min:id_max,:]
652 avgdata[:,i,:] = junk.sum(axis=1)
652 avgdata[:,i,:] = junk.sum(axis=1)
653 id_min += self.n
653 id_min += self.n
654 id_max += self.n
654 id_max += self.n
655
655
656 timeInterval = dataOut.ippSeconds*self.n
656 timeInterval = dataOut.ippSeconds*self.n
657 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
657 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
658 self.__dataReady = True
658 self.__dataReady = True
659 return avgdata, avgdatatime
659 return avgdata, avgdatatime
660
660
661 def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
661 def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
662
662
663 if not self.isConfig:
663 if not self.isConfig:
664 self.setup(n=n, stride=stride, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
664 self.setup(n=n, stride=stride, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
665 self.isConfig = True
665 self.isConfig = True
666
666
667 if dataOut.flagDataAsBlock:
667 if dataOut.flagDataAsBlock:
668 """
668 """
669 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
669 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
670 """
670 """
671 avgdata, avgdatatime = self.integrateByBlock(dataOut)
671 avgdata, avgdatatime = self.integrateByBlock(dataOut)
672 dataOut.nProfiles /= self.n
672 dataOut.nProfiles /= self.n
673 else:
673 else:
674 if stride is None:
674 if stride is None:
675 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
675 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
676 else:
676 else:
677 avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
677 avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
678
678
679
679
680 # dataOut.timeInterval *= n
680 # dataOut.timeInterval *= n
681 dataOut.flagNoData = True
681 dataOut.flagNoData = True
682
682
683 if self.__dataReady:
683 if self.__dataReady:
684 dataOut.data = avgdata
684 dataOut.data = avgdata
685 if not dataOut.flagCohInt:
685 if not dataOut.flagCohInt:
686 dataOut.nCohInt *= self.n
686 dataOut.nCohInt *= self.n
687 dataOut.flagCohInt = True
687 dataOut.flagCohInt = True
688 dataOut.utctime = avgdatatime
688 dataOut.utctime = avgdatatime
689 # print avgdata, avgdatatime
689 # print avgdata, avgdatatime
690 # raise
690 # raise
691 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
691 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
692 dataOut.flagNoData = False
692 dataOut.flagNoData = False
693 return dataOut
693 return dataOut
694
694
695 class Decoder(Operation):
695 class Decoder(Operation):
696
696
697 isConfig = False
697 isConfig = False
698 __profIndex = 0
698 __profIndex = 0
699
699
700 code = None
700 code = None
701
701
702 nCode = None
702 nCode = None
703 nBaud = None
703 nBaud = None
704
704
705 def __init__(self, **kwargs):
705 def __init__(self, **kwargs):
706
706
707 Operation.__init__(self, **kwargs)
707 Operation.__init__(self, **kwargs)
708
708
709 self.times = None
709 self.times = None
710 self.osamp = None
710 self.osamp = None
711 # self.__setValues = False
711 # self.__setValues = False
712 self.isConfig = False
712 self.isConfig = False
713 self.setupReq = False
713 self.setupReq = False
714 def setup(self, code, osamp, dataOut):
714 def setup(self, code, osamp, dataOut):
715
715
716 self.__profIndex = 0
716 self.__profIndex = 0
717
717
718 self.code = code
718 self.code = code
719
719
720 self.nCode = len(code)
720 self.nCode = len(code)
721 self.nBaud = len(code[0])
721 self.nBaud = len(code[0])
722 if (osamp != None) and (osamp >1):
722 if (osamp != None) and (osamp >1):
723 self.osamp = osamp
723 self.osamp = osamp
724 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
724 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
725 self.nBaud = self.nBaud*self.osamp
725 self.nBaud = self.nBaud*self.osamp
726
726
727 self.__nChannels = dataOut.nChannels
727 self.__nChannels = dataOut.nChannels
728 self.__nProfiles = dataOut.nProfiles
728 self.__nProfiles = dataOut.nProfiles
729 self.__nHeis = dataOut.nHeights
729 self.__nHeis = dataOut.nHeights
730
730
731 if self.__nHeis < self.nBaud:
731 if self.__nHeis < self.nBaud:
732 raise ValueError('Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud))
732 raise ValueError('Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud))
733
733
734 #Frequency
734 #Frequency
735 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
735 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
736
736
737 __codeBuffer[:,0:self.nBaud] = self.code
737 __codeBuffer[:,0:self.nBaud] = self.code
738
738
739 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
739 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
740
740
741 if dataOut.flagDataAsBlock:
741 if dataOut.flagDataAsBlock:
742
742
743 self.ndatadec = self.__nHeis #- self.nBaud + 1
743 self.ndatadec = self.__nHeis #- self.nBaud + 1
744
744
745 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
745 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
746
746
747 else:
747 else:
748
748
749 #Time
749 #Time
750 self.ndatadec = self.__nHeis #- self.nBaud + 1
750 self.ndatadec = self.__nHeis #- self.nBaud + 1
751
751
752 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
752 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
753
753
754 def __convolutionInFreq(self, data):
754 def __convolutionInFreq(self, data):
755
755
756 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
756 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
757
757
758 fft_data = numpy.fft.fft(data, axis=1)
758 fft_data = numpy.fft.fft(data, axis=1)
759
759
760 conv = fft_data*fft_code
760 conv = fft_data*fft_code
761
761
762 data = numpy.fft.ifft(conv,axis=1)
762 data = numpy.fft.ifft(conv,axis=1)
763
763
764 return data
764 return data
765
765
766 def __convolutionInFreqOpt(self, data):
766 def __convolutionInFreqOpt(self, data):
767
767
768 raise NotImplementedError
768 raise NotImplementedError
769
769
770 def __convolutionInTime(self, data):
770 def __convolutionInTime(self, data):
771
771
772 code = self.code[self.__profIndex]
772 code = self.code[self.__profIndex]
773 for i in range(self.__nChannels):
773 for i in range(self.__nChannels):
774 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
774 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
775
775
776 return self.datadecTime
776 return self.datadecTime
777
777
778 def __convolutionByBlockInTime(self, data):
778 def __convolutionByBlockInTime(self, data):
779
779
780 repetitions = int(self.__nProfiles / self.nCode)
780 repetitions = int(self.__nProfiles / self.nCode)
781 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
781 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
782 junk = junk.flatten()
782 junk = junk.flatten()
783 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
783 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
784 profilesList = range(self.__nProfiles)
784 profilesList = range(self.__nProfiles)
785
785
786 for i in range(self.__nChannels):
786 for i in range(self.__nChannels):
787 for j in profilesList:
787 for j in profilesList:
788 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
788 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
789 return self.datadecTime
789 return self.datadecTime
790
790
791 def __convolutionByBlockInFreq(self, data):
791 def __convolutionByBlockInFreq(self, data):
792
792
793 raise NotImplementedError("Decoder by frequency fro Blocks not implemented")
793 raise NotImplementedError("Decoder by frequency fro Blocks not implemented")
794
794
795
795
796 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
796 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
797
797
798 fft_data = numpy.fft.fft(data, axis=2)
798 fft_data = numpy.fft.fft(data, axis=2)
799
799
800 conv = fft_data*fft_code
800 conv = fft_data*fft_code
801
801
802 data = numpy.fft.ifft(conv,axis=2)
802 data = numpy.fft.ifft(conv,axis=2)
803
803
804 return data
804 return data
805
805
806
806
807 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
807 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
808
808
809 if dataOut.flagDecodeData:
809 if dataOut.flagDecodeData:
810 print("This data is already decoded, recoding again ...")
810 print("This data is already decoded, recoding again ...")
811
811
812 if not self.isConfig:
812 if not self.isConfig:
813
813
814 if code is None:
814 if code is None:
815 if dataOut.code is None:
815 if dataOut.code is None:
816 raise ValueError("Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type)
816 raise ValueError("Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type)
817
817
818 code = dataOut.code
818 code = dataOut.code
819 else:
819 else:
820 code = numpy.array(code).reshape(nCode,nBaud)
820 code = numpy.array(code).reshape(nCode,nBaud)
821 self.setup(code, osamp, dataOut)
821 self.setup(code, osamp, dataOut)
822
822
823 self.isConfig = True
823 self.isConfig = True
824
824
825 if mode == 3:
825 if mode == 3:
826 sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
826 sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
827
827
828 if times != None:
828 if times != None:
829 sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
829 sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
830
830
831 if self.code is None:
831 if self.code is None:
832 print("Fail decoding: Code is not defined.")
832 print("Fail decoding: Code is not defined.")
833 return
833 return
834
834
835 self.__nProfiles = dataOut.nProfiles
835 self.__nProfiles = dataOut.nProfiles
836 datadec = None
836 datadec = None
837
837
838 if mode == 3:
838 if mode == 3:
839 mode = 0
839 mode = 0
840
840
841 if dataOut.flagDataAsBlock:
841 if dataOut.flagDataAsBlock:
842 """
842 """
843 Decoding when data have been read as block,
843 Decoding when data have been read as block,
844 """
844 """
845
845
846 if mode == 0:
846 if mode == 0:
847 datadec = self.__convolutionByBlockInTime(dataOut.data)
847 datadec = self.__convolutionByBlockInTime(dataOut.data)
848 if mode == 1:
848 if mode == 1:
849 datadec = self.__convolutionByBlockInFreq(dataOut.data)
849 datadec = self.__convolutionByBlockInFreq(dataOut.data)
850 else:
850 else:
851 """
851 """
852 Decoding when data have been read profile by profile
852 Decoding when data have been read profile by profile
853 """
853 """
854 if mode == 0:
854 if mode == 0:
855 datadec = self.__convolutionInTime(dataOut.data)
855 datadec = self.__convolutionInTime(dataOut.data)
856
856
857 if mode == 1:
857 if mode == 1:
858 datadec = self.__convolutionInFreq(dataOut.data)
858 datadec = self.__convolutionInFreq(dataOut.data)
859
859
860 if mode == 2:
860 if mode == 2:
861 datadec = self.__convolutionInFreqOpt(dataOut.data)
861 datadec = self.__convolutionInFreqOpt(dataOut.data)
862
862
863 if datadec is None:
863 if datadec is None:
864 raise ValueError("Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode)
864 raise ValueError("Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode)
865
865
866 dataOut.code = self.code
866 dataOut.code = self.code
867 dataOut.nCode = self.nCode
867 dataOut.nCode = self.nCode
868 dataOut.nBaud = self.nBaud
868 dataOut.nBaud = self.nBaud
869
869
870 dataOut.data = datadec
870 dataOut.data = datadec
871
871
872 dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
872 dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
873
873
874 dataOut.flagDecodeData = True #asumo q la data esta decodificada
874 dataOut.flagDecodeData = True #asumo q la data esta decodificada
875
875
876 if self.__profIndex == self.nCode-1:
876 if self.__profIndex == self.nCode-1:
877 self.__profIndex = 0
877 self.__profIndex = 0
878 return dataOut
878 return dataOut
879
879
880 self.__profIndex += 1
880 self.__profIndex += 1
881
881
882 return dataOut
882 return dataOut
883 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
883 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
884
884
885
885
886 class ProfileConcat(Operation):
886 class ProfileConcat(Operation):
887
887
888 isConfig = False
888 isConfig = False
889 buffer = None
889 buffer = None
890
890
891 def __init__(self, **kwargs):
891 def __init__(self, **kwargs):
892
892
893 Operation.__init__(self, **kwargs)
893 Operation.__init__(self, **kwargs)
894 self.profileIndex = 0
894 self.profileIndex = 0
895
895
896 def reset(self):
896 def reset(self):
897 self.buffer = numpy.zeros_like(self.buffer)
897 self.buffer = numpy.zeros_like(self.buffer)
898 self.start_index = 0
898 self.start_index = 0
899 self.times = 1
899 self.times = 1
900
900
901 def setup(self, data, m, n=1):
901 def setup(self, data, m, n=1):
902 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
902 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
903 self.nHeights = data.shape[1]#.nHeights
903 self.nHeights = data.shape[1]#.nHeights
904 self.start_index = 0
904 self.start_index = 0
905 self.times = 1
905 self.times = 1
906
906
907 def concat(self, data):
907 def concat(self, data):
908
908
909 self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
909 self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
910 self.start_index = self.start_index + self.nHeights
910 self.start_index = self.start_index + self.nHeights
911
911
912 def run(self, dataOut, m):
912 def run(self, dataOut, m):
913 dataOut.flagNoData = True
913 dataOut.flagNoData = True
914
914
915 if not self.isConfig:
915 if not self.isConfig:
916 self.setup(dataOut.data, m, 1)
916 self.setup(dataOut.data, m, 1)
917 self.isConfig = True
917 self.isConfig = True
918
918
919 if dataOut.flagDataAsBlock:
919 if dataOut.flagDataAsBlock:
920 raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
920 raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
921
921
922 else:
922 else:
923 self.concat(dataOut.data)
923 self.concat(dataOut.data)
924 self.times += 1
924 self.times += 1
925 if self.times > m:
925 if self.times > m:
926 dataOut.data = self.buffer
926 dataOut.data = self.buffer
927 self.reset()
927 self.reset()
928 dataOut.flagNoData = False
928 dataOut.flagNoData = False
929 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
929 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
930 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
930 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
931 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
931 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
932 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
932 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
933 dataOut.ippSeconds *= m
933 dataOut.ippSeconds *= m
934 return dataOut
934 return dataOut
935
935
936 class ProfileSelector(Operation):
936 class ProfileSelector(Operation):
937
937
938 profileIndex = None
938 profileIndex = None
939 # Tamanho total de los perfiles
939 # Tamanho total de los perfiles
940 nProfiles = None
940 nProfiles = None
941
941
942 def __init__(self, **kwargs):
942 def __init__(self, **kwargs):
943
943
944 Operation.__init__(self, **kwargs)
944 Operation.__init__(self, **kwargs)
945 self.profileIndex = 0
945 self.profileIndex = 0
946
946
947 def incProfileIndex(self):
947 def incProfileIndex(self):
948
948
949 self.profileIndex += 1
949 self.profileIndex += 1
950
950
951 if self.profileIndex >= self.nProfiles:
951 if self.profileIndex >= self.nProfiles:
952 self.profileIndex = 0
952 self.profileIndex = 0
953
953
954 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
954 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
955
955
956 if profileIndex < minIndex:
956 if profileIndex < minIndex:
957 return False
957 return False
958
958
959 if profileIndex > maxIndex:
959 if profileIndex > maxIndex:
960 return False
960 return False
961
961
962 return True
962 return True
963
963
964 def isThisProfileInList(self, profileIndex, profileList):
964 def isThisProfileInList(self, profileIndex, profileList):
965
965
966 if profileIndex not in profileList:
966 if profileIndex not in profileList:
967 return False
967 return False
968
968
969 return True
969 return True
970
970
971 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
971 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
972
972
973 """
973 """
974 ProfileSelector:
974 ProfileSelector:
975
975
976 Inputs:
976 Inputs:
977 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
977 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
978
978
979 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
979 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
980
980
981 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
981 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
982
982
983 """
983 """
984
984
985 if rangeList is not None:
985 if rangeList is not None:
986 if type(rangeList[0]) not in (tuple, list):
986 if type(rangeList[0]) not in (tuple, list):
987 rangeList = [rangeList]
987 rangeList = [rangeList]
988
988
989 dataOut.flagNoData = True
989 dataOut.flagNoData = True
990
990
991 if dataOut.flagDataAsBlock:
991 if dataOut.flagDataAsBlock:
992 """
992 """
993 data dimension = [nChannels, nProfiles, nHeis]
993 data dimension = [nChannels, nProfiles, nHeis]
994 """
994 """
995 if profileList != None:
995 if profileList != None:
996 dataOut.data = dataOut.data[:,profileList,:]
996 dataOut.data = dataOut.data[:,profileList,:]
997
997
998 if profileRangeList != None:
998 if profileRangeList != None:
999 minIndex = profileRangeList[0]
999 minIndex = profileRangeList[0]
1000 maxIndex = profileRangeList[1]
1000 maxIndex = profileRangeList[1]
1001 profileList = list(range(minIndex, maxIndex+1))
1001 profileList = list(range(minIndex, maxIndex+1))
1002
1002
1003 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
1003 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
1004
1004
1005 if rangeList != None:
1005 if rangeList != None:
1006
1006
1007 profileList = []
1007 profileList = []
1008
1008
1009 for thisRange in rangeList:
1009 for thisRange in rangeList:
1010 minIndex = thisRange[0]
1010 minIndex = thisRange[0]
1011 maxIndex = thisRange[1]
1011 maxIndex = thisRange[1]
1012
1012
1013 profileList.extend(list(range(minIndex, maxIndex+1)))
1013 profileList.extend(list(range(minIndex, maxIndex+1)))
1014
1014
1015 dataOut.data = dataOut.data[:,profileList,:]
1015 dataOut.data = dataOut.data[:,profileList,:]
1016
1016
1017 dataOut.nProfiles = len(profileList)
1017 dataOut.nProfiles = len(profileList)
1018 dataOut.profileIndex = dataOut.nProfiles - 1
1018 dataOut.profileIndex = dataOut.nProfiles - 1
1019 dataOut.flagNoData = False
1019 dataOut.flagNoData = False
1020
1020
1021 return dataOut
1021 return dataOut
1022
1022
1023 """
1023 """
1024 data dimension = [nChannels, nHeis]
1024 data dimension = [nChannels, nHeis]
1025 """
1025 """
1026
1026
1027 if profileList != None:
1027 if profileList != None:
1028
1028
1029 if self.isThisProfileInList(dataOut.profileIndex, profileList):
1029 if self.isThisProfileInList(dataOut.profileIndex, profileList):
1030
1030
1031 self.nProfiles = len(profileList)
1031 self.nProfiles = len(profileList)
1032 dataOut.nProfiles = self.nProfiles
1032 dataOut.nProfiles = self.nProfiles
1033 dataOut.profileIndex = self.profileIndex
1033 dataOut.profileIndex = self.profileIndex
1034 dataOut.flagNoData = False
1034 dataOut.flagNoData = False
1035
1035
1036 self.incProfileIndex()
1036 self.incProfileIndex()
1037 return dataOut
1037 return dataOut
1038
1038
1039 if profileRangeList != None:
1039 if profileRangeList != None:
1040
1040
1041 minIndex = profileRangeList[0]
1041 minIndex = profileRangeList[0]
1042 maxIndex = profileRangeList[1]
1042 maxIndex = profileRangeList[1]
1043
1043
1044 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1044 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1045
1045
1046 self.nProfiles = maxIndex - minIndex + 1
1046 self.nProfiles = maxIndex - minIndex + 1
1047 dataOut.nProfiles = self.nProfiles
1047 dataOut.nProfiles = self.nProfiles
1048 dataOut.profileIndex = self.profileIndex
1048 dataOut.profileIndex = self.profileIndex
1049 dataOut.flagNoData = False
1049 dataOut.flagNoData = False
1050
1050
1051 self.incProfileIndex()
1051 self.incProfileIndex()
1052 return dataOut
1052 return dataOut
1053
1053
1054 if rangeList != None:
1054 if rangeList != None:
1055
1055
1056 nProfiles = 0
1056 nProfiles = 0
1057
1057
1058 for thisRange in rangeList:
1058 for thisRange in rangeList:
1059 minIndex = thisRange[0]
1059 minIndex = thisRange[0]
1060 maxIndex = thisRange[1]
1060 maxIndex = thisRange[1]
1061
1061
1062 nProfiles += maxIndex - minIndex + 1
1062 nProfiles += maxIndex - minIndex + 1
1063
1063
1064 for thisRange in rangeList:
1064 for thisRange in rangeList:
1065
1065
1066 minIndex = thisRange[0]
1066 minIndex = thisRange[0]
1067 maxIndex = thisRange[1]
1067 maxIndex = thisRange[1]
1068
1068
1069 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1069 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
1070
1070
1071 self.nProfiles = nProfiles
1071 self.nProfiles = nProfiles
1072 dataOut.nProfiles = self.nProfiles
1072 dataOut.nProfiles = self.nProfiles
1073 dataOut.profileIndex = self.profileIndex
1073 dataOut.profileIndex = self.profileIndex
1074 dataOut.flagNoData = False
1074 dataOut.flagNoData = False
1075
1075
1076 self.incProfileIndex()
1076 self.incProfileIndex()
1077
1077
1078 break
1078 break
1079
1079
1080 return dataOut
1080 return dataOut
1081
1081
1082
1082
1083 if beam != None: #beam is only for AMISR data
1083 if beam != None: #beam is only for AMISR data
1084 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
1084 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
1085 dataOut.flagNoData = False
1085 dataOut.flagNoData = False
1086 dataOut.profileIndex = self.profileIndex
1086 dataOut.profileIndex = self.profileIndex
1087
1087
1088 self.incProfileIndex()
1088 self.incProfileIndex()
1089
1089
1090 return dataOut
1090 return dataOut
1091
1091
1092 raise ValueError("ProfileSelector needs profileList, profileRangeList or rangeList parameter")
1092 raise ValueError("ProfileSelector needs profileList, profileRangeList or rangeList parameter")
1093
1093
1094
1094
1095 class Reshaper(Operation):
1095 class Reshaper(Operation):
1096
1096
1097 def __init__(self, **kwargs):
1097 def __init__(self, **kwargs):
1098
1098
1099 Operation.__init__(self, **kwargs)
1099 Operation.__init__(self, **kwargs)
1100
1100
1101 self.__buffer = None
1101 self.__buffer = None
1102 self.__nitems = 0
1102 self.__nitems = 0
1103
1103
1104 def __appendProfile(self, dataOut, nTxs):
1104 def __appendProfile(self, dataOut, nTxs):
1105
1105
1106 if self.__buffer is None:
1106 if self.__buffer is None:
1107 shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
1107 shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
1108 self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
1108 self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
1109
1109
1110 ini = dataOut.nHeights * self.__nitems
1110 ini = dataOut.nHeights * self.__nitems
1111 end = ini + dataOut.nHeights
1111 end = ini + dataOut.nHeights
1112
1112
1113 self.__buffer[:, ini:end] = dataOut.data
1113 self.__buffer[:, ini:end] = dataOut.data
1114
1114
1115 self.__nitems += 1
1115 self.__nitems += 1
1116
1116
1117 return int(self.__nitems*nTxs)
1117 return int(self.__nitems*nTxs)
1118
1118
1119 def __getBuffer(self):
1119 def __getBuffer(self):
1120
1120
1121 if self.__nitems == int(1./self.__nTxs):
1121 if self.__nitems == int(1./self.__nTxs):
1122
1122
1123 self.__nitems = 0
1123 self.__nitems = 0
1124
1124
1125 return self.__buffer.copy()
1125 return self.__buffer.copy()
1126
1126
1127 return None
1127 return None
1128
1128
1129 def __checkInputs(self, dataOut, shape, nTxs):
1129 def __checkInputs(self, dataOut, shape, nTxs):
1130
1130
1131 if shape is None and nTxs is None:
1131 if shape is None and nTxs is None:
1132 raise ValueError("Reshaper: shape of factor should be defined")
1132 raise ValueError("Reshaper: shape of factor should be defined")
1133
1133
1134 if nTxs:
1134 if nTxs:
1135 if nTxs < 0:
1135 if nTxs < 0:
1136 raise ValueError("nTxs should be greater than 0")
1136 raise ValueError("nTxs should be greater than 0")
1137
1137
1138 if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
1138 if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
1139 raise ValueError("nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs)))
1139 raise ValueError("nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs)))
1140
1140
1141 shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
1141 shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
1142
1142
1143 return shape, nTxs
1143 return shape, nTxs
1144
1144
1145 if len(shape) != 2 and len(shape) != 3:
1145 if len(shape) != 2 and len(shape) != 3:
1146 raise ValueError("shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights))
1146 raise ValueError("shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights))
1147
1147
1148 if len(shape) == 2:
1148 if len(shape) == 2:
1149 shape_tuple = [dataOut.nChannels]
1149 shape_tuple = [dataOut.nChannels]
1150 shape_tuple.extend(shape)
1150 shape_tuple.extend(shape)
1151 else:
1151 else:
1152 shape_tuple = list(shape)
1152 shape_tuple = list(shape)
1153
1153
1154 nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
1154 nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
1155
1155
1156 return shape_tuple, nTxs
1156 return shape_tuple, nTxs
1157
1157
1158 def run(self, dataOut, shape=None, nTxs=None):
1158 def run(self, dataOut, shape=None, nTxs=None):
1159
1159
1160 shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
1160 shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
1161
1161
1162 dataOut.flagNoData = True
1162 dataOut.flagNoData = True
1163 profileIndex = None
1163 profileIndex = None
1164
1164
1165 if dataOut.flagDataAsBlock:
1165 if dataOut.flagDataAsBlock:
1166
1166
1167 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
1167 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
1168 dataOut.flagNoData = False
1168 dataOut.flagNoData = False
1169
1169
1170 profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
1170 profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
1171
1171
1172 else:
1172 else:
1173
1173
1174 if self.__nTxs < 1:
1174 if self.__nTxs < 1:
1175
1175
1176 self.__appendProfile(dataOut, self.__nTxs)
1176 self.__appendProfile(dataOut, self.__nTxs)
1177 new_data = self.__getBuffer()
1177 new_data = self.__getBuffer()
1178
1178
1179 if new_data is not None:
1179 if new_data is not None:
1180 dataOut.data = new_data
1180 dataOut.data = new_data
1181 dataOut.flagNoData = False
1181 dataOut.flagNoData = False
1182
1182
1183 profileIndex = dataOut.profileIndex*nTxs
1183 profileIndex = dataOut.profileIndex*nTxs
1184
1184
1185 else:
1185 else:
1186 raise ValueError("nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)")
1186 raise ValueError("nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)")
1187
1187
1188 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1188 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1189
1189
1190 dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
1190 dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
1191
1191
1192 dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
1192 dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
1193
1193
1194 dataOut.profileIndex = profileIndex
1194 dataOut.profileIndex = profileIndex
1195
1195
1196 dataOut.ippSeconds /= self.__nTxs
1196 dataOut.ippSeconds /= self.__nTxs
1197
1197
1198 return dataOut
1198 return dataOut
1199
1199
1200 class SplitProfiles(Operation):
1200 class SplitProfiles(Operation):
1201
1201
1202 def __init__(self, **kwargs):
1202 def __init__(self, **kwargs):
1203
1203
1204 Operation.__init__(self, **kwargs)
1204 Operation.__init__(self, **kwargs)
1205
1205
1206 def run(self, dataOut, n):
1206 def run(self, dataOut, n):
1207
1207
1208 dataOut.flagNoData = True
1208 dataOut.flagNoData = True
1209 profileIndex = None
1209 profileIndex = None
1210
1210
1211 if dataOut.flagDataAsBlock:
1211 if dataOut.flagDataAsBlock:
1212
1212
1213 #nchannels, nprofiles, nsamples
1213 #nchannels, nprofiles, nsamples
1214 shape = dataOut.data.shape
1214 shape = dataOut.data.shape
1215
1215
1216 if shape[2] % n != 0:
1216 if shape[2] % n != 0:
1217 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
1217 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
1218
1218
1219 new_shape = shape[0], shape[1]*n, int(shape[2]/n)
1219 new_shape = shape[0], shape[1]*n, int(shape[2]/n)
1220
1220
1221 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1221 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1222 dataOut.flagNoData = False
1222 dataOut.flagNoData = False
1223
1223
1224 profileIndex = int(dataOut.nProfiles/n) - 1
1224 profileIndex = int(dataOut.nProfiles/n) - 1
1225
1225
1226 else:
1226 else:
1227
1227
1228 raise ValueError("Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)")
1228 raise ValueError("Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)")
1229
1229
1230 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1230 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1231
1231
1232 dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
1232 dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
1233
1233
1234 dataOut.nProfiles = int(dataOut.nProfiles*n)
1234 dataOut.nProfiles = int(dataOut.nProfiles*n)
1235
1235
1236 dataOut.profileIndex = profileIndex
1236 dataOut.profileIndex = profileIndex
1237
1237
1238 dataOut.ippSeconds /= n
1238 dataOut.ippSeconds /= n
1239
1239
1240 return dataOut
1240 return dataOut
1241
1241
1242 class CombineProfiles(Operation):
1242 class CombineProfiles(Operation):
1243 def __init__(self, **kwargs):
1243 def __init__(self, **kwargs):
1244
1244
1245 Operation.__init__(self, **kwargs)
1245 Operation.__init__(self, **kwargs)
1246
1246
1247 self.__remData = None
1247 self.__remData = None
1248 self.__profileIndex = 0
1248 self.__profileIndex = 0
1249
1249
1250 def run(self, dataOut, n):
1250 def run(self, dataOut, n):
1251
1251
1252 dataOut.flagNoData = True
1252 dataOut.flagNoData = True
1253 profileIndex = None
1253 profileIndex = None
1254
1254
1255 if dataOut.flagDataAsBlock:
1255 if dataOut.flagDataAsBlock:
1256
1256
1257 #nchannels, nprofiles, nsamples
1257 #nchannels, nprofiles, nsamples
1258 shape = dataOut.data.shape
1258 shape = dataOut.data.shape
1259 new_shape = shape[0], shape[1]/n, shape[2]*n
1259 new_shape = shape[0], shape[1]/n, shape[2]*n
1260
1260
1261 if shape[1] % n != 0:
1261 if shape[1] % n != 0:
1262 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[1]))
1262 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[1]))
1263
1263
1264 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1264 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1265 dataOut.flagNoData = False
1265 dataOut.flagNoData = False
1266
1266
1267 profileIndex = int(dataOut.nProfiles*n) - 1
1267 profileIndex = int(dataOut.nProfiles*n) - 1
1268
1268
1269 else:
1269 else:
1270
1270
1271 #nchannels, nsamples
1271 #nchannels, nsamples
1272 if self.__remData is None:
1272 if self.__remData is None:
1273 newData = dataOut.data
1273 newData = dataOut.data
1274 else:
1274 else:
1275 newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
1275 newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
1276
1276
1277 self.__profileIndex += 1
1277 self.__profileIndex += 1
1278
1278
1279 if self.__profileIndex < n:
1279 if self.__profileIndex < n:
1280 self.__remData = newData
1280 self.__remData = newData
1281 #continue
1281 #continue
1282 return
1282 return
1283
1283
1284 self.__profileIndex = 0
1284 self.__profileIndex = 0
1285 self.__remData = None
1285 self.__remData = None
1286
1286
1287 dataOut.data = newData
1287 dataOut.data = newData
1288 dataOut.flagNoData = False
1288 dataOut.flagNoData = False
1289
1289
1290 profileIndex = dataOut.profileIndex/n
1290 profileIndex = dataOut.profileIndex/n
1291
1291
1292
1292
1293 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1293 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1294
1294
1295 dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
1295 dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
1296
1296
1297 dataOut.nProfiles = int(dataOut.nProfiles/n)
1297 dataOut.nProfiles = int(dataOut.nProfiles/n)
1298
1298
1299 dataOut.profileIndex = profileIndex
1299 dataOut.profileIndex = profileIndex
1300
1300
1301 dataOut.ippSeconds *= n
1301 dataOut.ippSeconds *= n
1302
1302
1303 return dataOut
1303 return dataOut
1304
1304
1305 class PulsePairVoltage(Operation):
1305 class PulsePairVoltage(Operation):
1306 '''
1306 '''
1307 Function PulsePair(Signal Power, Velocity)
1307 Function PulsePair(Signal Power, Velocity)
1308 The real component of Lag[0] provides Intensity Information
1308 The real component of Lag[0] provides Intensity Information
1309 The imag component of Lag[1] Phase provides Velocity Information
1309 The imag component of Lag[1] Phase provides Velocity Information
1310
1310
1311 Configuration Parameters:
1311 Configuration Parameters:
1312 nPRF = Number of Several PRF
1312 nPRF = Number of Several PRF
1313 theta = Degree Azimuth angel Boundaries
1313 theta = Degree Azimuth angel Boundaries
1314
1314
1315 Input:
1315 Input:
1316 self.dataOut
1316 self.dataOut
1317 lag[N]
1317 lag[N]
1318 Affected:
1318 Affected:
1319 self.dataOut.spc
1319 self.dataOut.spc
1320 '''
1320 '''
1321 isConfig = False
1321 isConfig = False
1322 __profIndex = 0
1322 __profIndex = 0
1323 __initime = None
1323 __initime = None
1324 __lastdatatime = None
1324 __lastdatatime = None
1325 __buffer = None
1325 __buffer = None
1326 noise = None
1326 noise = None
1327 __dataReady = False
1327 __dataReady = False
1328 n = None
1328 n = None
1329 __nch = 0
1329 __nch = 0
1330 __nHeis = 0
1330 __nHeis = 0
1331 removeDC = False
1331 removeDC = False
1332 ipp = None
1332 ipp = None
1333 lambda_ = 0
1333 lambda_ = 0
1334
1334
1335 def __init__(self,**kwargs):
1335 def __init__(self,**kwargs):
1336 Operation.__init__(self,**kwargs)
1336 Operation.__init__(self,**kwargs)
1337
1337
1338 def setup(self, dataOut, n = None, removeDC=False):
1338 def setup(self, dataOut, n = None, removeDC=False):
1339 '''
1339 '''
1340 n= Numero de PRF's de entrada
1340 n= Numero de PRF's de entrada
1341 '''
1341 '''
1342 self.__initime = None
1342 self.__initime = None
1343 self.__lastdatatime = 0
1343 self.__lastdatatime = 0
1344 self.__dataReady = False
1344 self.__dataReady = False
1345 self.__buffer = 0
1345 self.__buffer = 0
1346 self.__profIndex = 0
1346 self.__profIndex = 0
1347 self.noise = None
1347 self.noise = None
1348 self.__nch = dataOut.nChannels
1348 self.__nch = dataOut.nChannels
1349 self.__nHeis = dataOut.nHeights
1349 self.__nHeis = dataOut.nHeights
1350 self.removeDC = removeDC
1350 self.removeDC = removeDC
1351 self.lambda_ = 3.0e8/(9345.0e6)
1351 self.lambda_ = 3.0e8/(9345.0e6)
1352 self.ippSec = dataOut.ippSeconds
1352 self.ippSec = dataOut.ippSeconds
1353 self.nCohInt = dataOut.nCohInt
1353 self.nCohInt = dataOut.nCohInt
1354
1354
1355 if n == None:
1355 if n == None:
1356 raise ValueError("n should be specified.")
1356 raise ValueError("n should be specified.")
1357
1357
1358 if n != None:
1358 if n != None:
1359 if n<2:
1359 if n<2:
1360 raise ValueError("n should be greater than 2")
1360 raise ValueError("n should be greater than 2")
1361
1361
1362 self.n = n
1362 self.n = n
1363 self.__nProf = n
1363 self.__nProf = n
1364
1364
1365 self.__buffer = numpy.zeros((dataOut.nChannels,
1365 self.__buffer = numpy.zeros((dataOut.nChannels,
1366 n,
1366 n,
1367 dataOut.nHeights),
1367 dataOut.nHeights),
1368 dtype='complex')
1368 dtype='complex')
1369
1369
1370 def putData(self,data):
1370 def putData(self,data):
1371 '''
1371 '''
1372 Add a profile to he __buffer and increase in one the __profiel Index
1372 Add a profile to he __buffer and increase in one the __profiel Index
1373 '''
1373 '''
1374 self.__buffer[:,self.__profIndex,:]= data
1374 self.__buffer[:,self.__profIndex,:]= data
1375 self.__profIndex += 1
1375 self.__profIndex += 1
1376 return
1376 return
1377
1377
1378 def pushData(self,dataOut):
1378 def pushData(self,dataOut):
1379 '''
1379 '''
1380 Return the PULSEPAIR and the profiles used in the operation
1380 Return the PULSEPAIR and the profiles used in the operation
1381 Affected : self.__profileIndex
1381 Affected : self.__profileIndex
1382 '''
1382 '''
1383 #----------------- Remove DC-----------------------------------
1383 #----------------- Remove DC-----------------------------------
1384 if self.removeDC==True:
1384 if self.removeDC==True:
1385 mean = numpy.mean(self.__buffer,1)
1385 mean = numpy.mean(self.__buffer,1)
1386 tmp = mean.reshape(self.__nch,1,self.__nHeis)
1386 tmp = mean.reshape(self.__nch,1,self.__nHeis)
1387 dc= numpy.tile(tmp,[1,self.__nProf,1])
1387 dc= numpy.tile(tmp,[1,self.__nProf,1])
1388 self.__buffer = self.__buffer - dc
1388 self.__buffer = self.__buffer - dc
1389 #------------------Calculo de Potencia ------------------------
1389 #------------------Calculo de Potencia ------------------------
1390 pair0 = self.__buffer*numpy.conj(self.__buffer)
1390 pair0 = self.__buffer*numpy.conj(self.__buffer)
1391 pair0 = pair0.real
1391 pair0 = pair0.real
1392 lag_0 = numpy.sum(pair0,1)
1392 lag_0 = numpy.sum(pair0,1)
1393 #------------------Calculo de Ruido x canal--------------------
1393 #------------------Calculo de Ruido x canal--------------------
1394 self.noise = numpy.zeros(self.__nch)
1394 self.noise = numpy.zeros(self.__nch)
1395 for i in range(self.__nch):
1395 for i in range(self.__nch):
1396 daux = numpy.sort(pair0[i,:,:],axis= None)
1396 daux = numpy.sort(pair0[i,:,:],axis= None)
1397 self.noise[i]=hildebrand_sekhon( daux ,self.nCohInt)
1397 self.noise[i]=hildebrand_sekhon( daux ,self.nCohInt)
1398
1398
1399 self.noise = self.noise.reshape(self.__nch,1)
1399 self.noise = self.noise.reshape(self.__nch,1)
1400 self.noise = numpy.tile(self.noise,[1,self.__nHeis])
1400 self.noise = numpy.tile(self.noise,[1,self.__nHeis])
1401 noise_buffer = self.noise.reshape(self.__nch,1,self.__nHeis)
1401 noise_buffer = self.noise.reshape(self.__nch,1,self.__nHeis)
1402 noise_buffer = numpy.tile(noise_buffer,[1,self.__nProf,1])
1402 noise_buffer = numpy.tile(noise_buffer,[1,self.__nProf,1])
1403 #------------------ Potencia recibida= P , Potencia senal = S , Ruido= N--
1403 #------------------ Potencia recibida= P , Potencia senal = S , Ruido= N--
1404 #------------------ P= S+N ,P=lag_0/N ---------------------------------
1404 #------------------ P= S+N ,P=lag_0/N ---------------------------------
1405 #-------------------- Power --------------------------------------------------
1405 #-------------------- Power --------------------------------------------------
1406 data_power = lag_0/(self.n*self.nCohInt)
1406 data_power = lag_0/(self.n*self.nCohInt)
1407 #------------------ Senal ---------------------------------------------------
1407 #------------------ Senal ---------------------------------------------------
1408 data_intensity = pair0 - noise_buffer
1408 data_intensity = pair0 - noise_buffer
1409 data_intensity = numpy.sum(data_intensity,axis=1)*(self.n*self.nCohInt)#*self.nCohInt)
1409 data_intensity = numpy.sum(data_intensity,axis=1)*(self.n*self.nCohInt)#*self.nCohInt)
1410 #data_intensity = (lag_0-self.noise*self.n)*(self.n*self.nCohInt)
1410 #data_intensity = (lag_0-self.noise*self.n)*(self.n*self.nCohInt)
1411 for i in range(self.__nch):
1411 for i in range(self.__nch):
1412 for j in range(self.__nHeis):
1412 for j in range(self.__nHeis):
1413 if data_intensity[i][j] < 0:
1413 if data_intensity[i][j] < 0:
1414 data_intensity[i][j] = numpy.min(numpy.absolute(data_intensity[i][j]))
1414 data_intensity[i][j] = numpy.min(numpy.absolute(data_intensity[i][j]))
1415
1415
1416 #----------------- Calculo de Frecuencia y Velocidad doppler--------
1416 #----------------- Calculo de Frecuencia y Velocidad doppler--------
1417 pair1 = self.__buffer[:,:-1,:]*numpy.conjugate(self.__buffer[:,1:,:])
1417 pair1 = self.__buffer[:,:-1,:]*numpy.conjugate(self.__buffer[:,1:,:])
1418 lag_1 = numpy.sum(pair1,1)
1418 lag_1 = numpy.sum(pair1,1)
1419 data_freq = (-1/(2.0*math.pi*self.ippSec*self.nCohInt))*numpy.angle(lag_1)
1419 data_freq = (-1/(2.0*math.pi*self.ippSec*self.nCohInt))*numpy.angle(lag_1)
1420 data_velocity = (self.lambda_/2.0)*data_freq
1420 data_velocity = (self.lambda_/2.0)*data_freq
1421
1421
1422 #---------------- Potencia promedio estimada de la Senal-----------
1422 #---------------- Potencia promedio estimada de la Senal-----------
1423 lag_0 = lag_0/self.n
1423 lag_0 = lag_0/self.n
1424 S = lag_0-self.noise
1424 S = lag_0-self.noise
1425
1425
1426 #---------------- Frecuencia Doppler promedio ---------------------
1426 #---------------- Frecuencia Doppler promedio ---------------------
1427 lag_1 = lag_1/(self.n-1)
1427 lag_1 = lag_1/(self.n-1)
1428 R1 = numpy.abs(lag_1)
1428 R1 = numpy.abs(lag_1)
1429
1429
1430 #---------------- Calculo del SNR----------------------------------
1430 #---------------- Calculo del SNR----------------------------------
1431 data_snrPP = S/self.noise
1431 data_snrPP = S/self.noise
1432 for i in range(self.__nch):
1432 for i in range(self.__nch):
1433 for j in range(self.__nHeis):
1433 for j in range(self.__nHeis):
1434 if data_snrPP[i][j] < 1.e-20:
1434 if data_snrPP[i][j] < 1.e-20:
1435 data_snrPP[i][j] = 1.e-20
1435 data_snrPP[i][j] = 1.e-20
1436
1436
1437 #----------------- Calculo del ancho espectral ----------------------
1437 #----------------- Calculo del ancho espectral ----------------------
1438 L = S/R1
1438 L = S/R1
1439 L = numpy.where(L<0,1,L)
1439 L = numpy.where(L<0,1,L)
1440 L = numpy.log(L)
1440 L = numpy.log(L)
1441 tmp = numpy.sqrt(numpy.absolute(L))
1441 tmp = numpy.sqrt(numpy.absolute(L))
1442 data_specwidth = (self.lambda_/(2*math.sqrt(2)*math.pi*self.ippSec*self.nCohInt))*tmp*numpy.sign(L)
1442 data_specwidth = (self.lambda_/(2*math.sqrt(2)*math.pi*self.ippSec*self.nCohInt))*tmp*numpy.sign(L)
1443 n = self.__profIndex
1443 n = self.__profIndex
1444
1444
1445 self.__buffer = numpy.zeros((self.__nch, self.__nProf,self.__nHeis), dtype='complex')
1445 self.__buffer = numpy.zeros((self.__nch, self.__nProf,self.__nHeis), dtype='complex')
1446 self.__profIndex = 0
1446 self.__profIndex = 0
1447 return data_power,data_intensity,data_velocity,data_snrPP,data_specwidth,n
1447 return data_power,data_intensity,data_velocity,data_snrPP,data_specwidth,n
1448
1448
1449
1449
1450 def pulsePairbyProfiles(self,dataOut):
1450 def pulsePairbyProfiles(self,dataOut):
1451
1451
1452 self.__dataReady = False
1452 self.__dataReady = False
1453 data_power = None
1453 data_power = None
1454 data_intensity = None
1454 data_intensity = None
1455 data_velocity = None
1455 data_velocity = None
1456 data_specwidth = None
1456 data_specwidth = None
1457 data_snrPP = None
1457 data_snrPP = None
1458 self.putData(data=dataOut.data)
1458 self.putData(data=dataOut.data)
1459 if self.__profIndex == self.n:
1459 if self.__profIndex == self.n:
1460 data_power,data_intensity, data_velocity,data_snrPP,data_specwidth, n = self.pushData(dataOut=dataOut)
1460 data_power,data_intensity, data_velocity,data_snrPP,data_specwidth, n = self.pushData(dataOut=dataOut)
1461 self.__dataReady = True
1461 self.__dataReady = True
1462
1462
1463 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth
1463 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth
1464
1464
1465
1465
1466 def pulsePairOp(self, dataOut, datatime= None):
1466 def pulsePairOp(self, dataOut, datatime= None):
1467
1467
1468 if self.__initime == None:
1468 if self.__initime == None:
1469 self.__initime = datatime
1469 self.__initime = datatime
1470 data_power, data_intensity, data_velocity, data_snrPP, data_specwidth = self.pulsePairbyProfiles(dataOut)
1470 data_power, data_intensity, data_velocity, data_snrPP, data_specwidth = self.pulsePairbyProfiles(dataOut)
1471 self.__lastdatatime = datatime
1471 self.__lastdatatime = datatime
1472
1472
1473 if data_power is None:
1473 if data_power is None:
1474 return None, None, None,None,None,None
1474 return None, None, None,None,None,None
1475
1475
1476 avgdatatime = self.__initime
1476 avgdatatime = self.__initime
1477 deltatime = datatime - self.__lastdatatime
1477 deltatime = datatime - self.__lastdatatime
1478 self.__initime = datatime
1478 self.__initime = datatime
1479
1479
1480 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth, avgdatatime
1480 return data_power, data_intensity, data_velocity, data_snrPP, data_specwidth, avgdatatime
1481
1481
1482 def run(self, dataOut,n = None,removeDC= False, overlapping= False,**kwargs):
1482 def run(self, dataOut,n = None,removeDC= False, overlapping= False,**kwargs):
1483
1483
1484 if not self.isConfig:
1484 if not self.isConfig:
1485 self.setup(dataOut = dataOut, n = n , removeDC=removeDC , **kwargs)
1485 self.setup(dataOut = dataOut, n = n , removeDC=removeDC , **kwargs)
1486 self.isConfig = True
1486 self.isConfig = True
1487 data_power, data_intensity, data_velocity,data_snrPP,data_specwidth, avgdatatime = self.pulsePairOp(dataOut, dataOut.utctime)
1487 data_power, data_intensity, data_velocity,data_snrPP,data_specwidth, avgdatatime = self.pulsePairOp(dataOut, dataOut.utctime)
1488 dataOut.flagNoData = True
1488 dataOut.flagNoData = True
1489
1489
1490 if self.__dataReady:
1490 if self.__dataReady:
1491 dataOut.nCohInt *= self.n
1491 dataOut.nCohInt *= self.n
1492 dataOut.dataPP_POW = data_intensity # S
1492 dataOut.dataPP_POW = data_intensity # S
1493 dataOut.dataPP_POWER = data_power # P
1493 dataOut.dataPP_POWER = data_power # P
1494 dataOut.dataPP_DOP = data_velocity
1494 dataOut.dataPP_DOP = data_velocity
1495 dataOut.dataPP_SNR = data_snrPP
1495 dataOut.dataPP_SNR = data_snrPP
1496 dataOut.dataPP_WIDTH = data_specwidth
1496 dataOut.dataPP_WIDTH = data_specwidth
1497 dataOut.PRFbyAngle = self.n #numero de PRF*cada angulo rotado que equivale a un tiempo.
1497 dataOut.PRFbyAngle = self.n #numero de PRF*cada angulo rotado que equivale a un tiempo.
1498 dataOut.utctime = avgdatatime
1498 dataOut.utctime = avgdatatime
1499 dataOut.flagNoData = False
1499 dataOut.flagNoData = False
1500 return dataOut
1500 return dataOut
1501
1501
1502
1502
1503
1503
1504 # import collections
1504 # import collections
1505 # from scipy.stats import mode
1505 # from scipy.stats import mode
1506 #
1506 #
1507 # class Synchronize(Operation):
1507 # class Synchronize(Operation):
1508 #
1508 #
1509 # isConfig = False
1509 # isConfig = False
1510 # __profIndex = 0
1510 # __profIndex = 0
1511 #
1511 #
1512 # def __init__(self, **kwargs):
1512 # def __init__(self, **kwargs):
1513 #
1513 #
1514 # Operation.__init__(self, **kwargs)
1514 # Operation.__init__(self, **kwargs)
1515 # # self.isConfig = False
1515 # # self.isConfig = False
1516 # self.__powBuffer = None
1516 # self.__powBuffer = None
1517 # self.__startIndex = 0
1517 # self.__startIndex = 0
1518 # self.__pulseFound = False
1518 # self.__pulseFound = False
1519 #
1519 #
1520 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
1520 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
1521 #
1521 #
1522 # #Read data
1522 # #Read data
1523 #
1523 #
1524 # powerdB = dataOut.getPower(channel = channel)
1524 # powerdB = dataOut.getPower(channel = channel)
1525 # noisedB = dataOut.getNoise(channel = channel)[0]
1525 # noisedB = dataOut.getNoise(channel = channel)[0]
1526 #
1526 #
1527 # self.__powBuffer.extend(powerdB.flatten())
1527 # self.__powBuffer.extend(powerdB.flatten())
1528 #
1528 #
1529 # dataArray = numpy.array(self.__powBuffer)
1529 # dataArray = numpy.array(self.__powBuffer)
1530 #
1530 #
1531 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
1531 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
1532 #
1532 #
1533 # maxValue = numpy.nanmax(filteredPower)
1533 # maxValue = numpy.nanmax(filteredPower)
1534 #
1534 #
1535 # if maxValue < noisedB + 10:
1535 # if maxValue < noisedB + 10:
1536 # #No se encuentra ningun pulso de transmision
1536 # #No se encuentra ningun pulso de transmision
1537 # return None
1537 # return None
1538 #
1538 #
1539 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
1539 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
1540 #
1540 #
1541 # if len(maxValuesIndex) < 2:
1541 # if len(maxValuesIndex) < 2:
1542 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
1542 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
1543 # return None
1543 # return None
1544 #
1544 #
1545 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
1545 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
1546 #
1546 #
1547 # #Seleccionar solo valores con un espaciamiento de nSamples
1547 # #Seleccionar solo valores con un espaciamiento de nSamples
1548 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
1548 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
1549 #
1549 #
1550 # if len(pulseIndex) < 2:
1550 # if len(pulseIndex) < 2:
1551 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1551 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1552 # return None
1552 # return None
1553 #
1553 #
1554 # spacing = pulseIndex[1:] - pulseIndex[:-1]
1554 # spacing = pulseIndex[1:] - pulseIndex[:-1]
1555 #
1555 #
1556 # #remover senales que se distancien menos de 10 unidades o muestras
1556 # #remover senales que se distancien menos de 10 unidades o muestras
1557 # #(No deberian existir IPP menor a 10 unidades)
1557 # #(No deberian existir IPP menor a 10 unidades)
1558 #
1558 #
1559 # realIndex = numpy.where(spacing > 10 )[0]
1559 # realIndex = numpy.where(spacing > 10 )[0]
1560 #
1560 #
1561 # if len(realIndex) < 2:
1561 # if len(realIndex) < 2:
1562 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1562 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1563 # return None
1563 # return None
1564 #
1564 #
1565 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
1565 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
1566 # realPulseIndex = pulseIndex[realIndex]
1566 # realPulseIndex = pulseIndex[realIndex]
1567 #
1567 #
1568 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
1568 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
1569 #
1569 #
1570 # print "IPP = %d samples" %period
1570 # print "IPP = %d samples" %period
1571 #
1571 #
1572 # self.__newNSamples = dataOut.nHeights #int(period)
1572 # self.__newNSamples = dataOut.nHeights #int(period)
1573 # self.__startIndex = int(realPulseIndex[0])
1573 # self.__startIndex = int(realPulseIndex[0])
1574 #
1574 #
1575 # return 1
1575 # return 1
1576 #
1576 #
1577 #
1577 #
1578 # def setup(self, nSamples, nChannels, buffer_size = 4):
1578 # def setup(self, nSamples, nChannels, buffer_size = 4):
1579 #
1579 #
1580 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1580 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1581 # maxlen = buffer_size*nSamples)
1581 # maxlen = buffer_size*nSamples)
1582 #
1582 #
1583 # bufferList = []
1583 # bufferList = []
1584 #
1584 #
1585 # for i in range(nChannels):
1585 # for i in range(nChannels):
1586 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1586 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1587 # maxlen = buffer_size*nSamples)
1587 # maxlen = buffer_size*nSamples)
1588 #
1588 #
1589 # bufferList.append(bufferByChannel)
1589 # bufferList.append(bufferByChannel)
1590 #
1590 #
1591 # self.__nSamples = nSamples
1591 # self.__nSamples = nSamples
1592 # self.__nChannels = nChannels
1592 # self.__nChannels = nChannels
1593 # self.__bufferList = bufferList
1593 # self.__bufferList = bufferList
1594 #
1594 #
1595 # def run(self, dataOut, channel = 0):
1595 # def run(self, dataOut, channel = 0):
1596 #
1596 #
1597 # if not self.isConfig:
1597 # if not self.isConfig:
1598 # nSamples = dataOut.nHeights
1598 # nSamples = dataOut.nHeights
1599 # nChannels = dataOut.nChannels
1599 # nChannels = dataOut.nChannels
1600 # self.setup(nSamples, nChannels)
1600 # self.setup(nSamples, nChannels)
1601 # self.isConfig = True
1601 # self.isConfig = True
1602 #
1602 #
1603 # #Append new data to internal buffer
1603 # #Append new data to internal buffer
1604 # for thisChannel in range(self.__nChannels):
1604 # for thisChannel in range(self.__nChannels):
1605 # bufferByChannel = self.__bufferList[thisChannel]
1605 # bufferByChannel = self.__bufferList[thisChannel]
1606 # bufferByChannel.extend(dataOut.data[thisChannel])
1606 # bufferByChannel.extend(dataOut.data[thisChannel])
1607 #
1607 #
1608 # if self.__pulseFound:
1608 # if self.__pulseFound:
1609 # self.__startIndex -= self.__nSamples
1609 # self.__startIndex -= self.__nSamples
1610 #
1610 #
1611 # #Finding Tx Pulse
1611 # #Finding Tx Pulse
1612 # if not self.__pulseFound:
1612 # if not self.__pulseFound:
1613 # indexFound = self.__findTxPulse(dataOut, channel)
1613 # indexFound = self.__findTxPulse(dataOut, channel)
1614 #
1614 #
1615 # if indexFound == None:
1615 # if indexFound == None:
1616 # dataOut.flagNoData = True
1616 # dataOut.flagNoData = True
1617 # return
1617 # return
1618 #
1618 #
1619 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1619 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1620 # self.__pulseFound = True
1620 # self.__pulseFound = True
1621 # self.__startIndex = indexFound
1621 # self.__startIndex = indexFound
1622 #
1622 #
1623 # #If pulse was found ...
1623 # #If pulse was found ...
1624 # for thisChannel in range(self.__nChannels):
1624 # for thisChannel in range(self.__nChannels):
1625 # bufferByChannel = self.__bufferList[thisChannel]
1625 # bufferByChannel = self.__bufferList[thisChannel]
1626 # #print self.__startIndex
1626 # #print self.__startIndex
1627 # x = numpy.array(bufferByChannel)
1627 # x = numpy.array(bufferByChannel)
1628 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1628 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1629 #
1629 #
1630 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1630 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1631 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1631 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1632 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1632 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1633 #
1633 #
1634 # dataOut.data = self.__arrayBuffer
1634 # dataOut.data = self.__arrayBuffer
1635 #
1635 #
1636 # self.__startIndex += self.__newNSamples
1636 # self.__startIndex += self.__newNSamples
1637 #
1637 #
1638 # return
1638 # return
1639 class SSheightProfiles(Operation):
1640
1641 step = None
1642 nsamples = None
1643 bufferShape = None
1644 profileShape = None
1645 sshProfiles = None
1646 profileIndex = None
1647
1648 def __init__(self, **kwargs):
1649
1650 Operation.__init__(self, **kwargs)
1651 self.isConfig = False
1652
1653 def setup(self,dataOut ,step = None , nsamples = None):
1654
1655 if step == None and nsamples == None:
1656 raise ValueError("step or nheights should be specified ...")
1657
1658 self.step = step
1659 self.nsamples = nsamples
1660 self.__nChannels = dataOut.nChannels
1661 self.__nProfiles = dataOut.nProfiles
1662 self.__nHeis = dataOut.nHeights
1663 shape = dataOut.data.shape #nchannels, nprofiles, nsamples
1664
1665 residue = (shape[1] - self.nsamples) % self.step
1666 if residue != 0:
1667 print("The residue is %d, step=%d should be multiple of %d to avoid loss of %d samples"%(residue,step,shape[1] - self.nsamples,residue))
1668
1669 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1670 numberProfile = self.nsamples
1671 numberSamples = (shape[1] - self.nsamples)/self.step
1672
1673 self.bufferShape = int(shape[0]), int(numberSamples), int(numberProfile) # nchannels, nsamples , nprofiles
1674 self.profileShape = int(shape[0]), int(numberProfile), int(numberSamples) # nchannels, nprofiles, nsamples
1675
1676 self.buffer = numpy.zeros(self.bufferShape , dtype=numpy.complex)
1677 self.sshProfiles = numpy.zeros(self.profileShape, dtype=numpy.complex)
1678
1679 def run(self, dataOut, step, nsamples, code = None, repeat = None):
1680 dataOut.flagNoData = True
1681
1682 profileIndex = None
1683 #print(dataOut.getFreqRange(1)/1000.)
1684 #exit(1)
1685 if dataOut.flagDataAsBlock:
1686 dataOut.data = numpy.average(dataOut.data,axis=1)
1687 #print("jee")
1688 dataOut.flagDataAsBlock = False
1689 if not self.isConfig:
1690 self.setup(dataOut, step=step , nsamples=nsamples)
1691 #print("Setup done")
1692 self.isConfig = True
1693
1694 #DC_Hae = numpy.array([0.398+0.588j, -0.926+0.306j, -0.536-0.682j, -0.072+0.53j, 0.368-0.356j, 0.996+0.362j])
1695 #DC_Hae = numpy.array([ 0.001025 +0.0516375j, 0.03485 +0.20923125j, -0.168 -0.02720625j,
1696 #-0.1105375 +0.0707125j, -0.20309375-0.09670625j, 0.189775 +0.02716875j])*(-3.5)
1697
1698 #DC_Hae = numpy.array([ -32.26 +8.66j, -32.26 +8.66j])
1699
1700 #DC_Hae = numpy.array([-2.78500000e-01 -1.39175j, -6.63237294e+02+210.4268625j])
1701
1702 #print(dataOut.data[0,13:15])
1703 #dataOut.data = dataOut.data - DC_Hae[:,None]
1704 #print(dataOut.data[0,13:15])
1705 #exit(1)
1706 if code is not None:
1707 code = numpy.array(code)
1708 code_block = code
1709 '''
1710 roll = 0
1711 code = numpy.roll(code,roll,axis=0)
1712 code = numpy.reshape(code,(5,100,64))
1713 block = dataOut.CurrentBlock%5
1714
1715 day_dif = 0 #day_19_Oct_2021: 3
1716 code_block = code[block-1-day_dif,:,:]
1717 '''
1718 if repeat is not None:
1719 code_block = numpy.repeat(code_block, repeats=repeat, axis=1)
1720 #print(code_block.shape)
1721 for i in range(self.buffer.shape[1]):
1722
1723 if code is not None:
1724 self.buffer[:,i] = dataOut.data[:,i*self.step:i*self.step + self.nsamples]*code_block
1725
1726 else:
1727
1728 self.buffer[:,i] = dataOut.data[:,i*self.step:i*self.step + self.nsamples]#*code[dataOut.profileIndex,:]
1729
1730 #self.buffer[:,j,self.__nHeis-j*self.step - self.nheights:self.__nHeis-j*self.step] = numpy.flip(dataOut.data[:,j*self.step:j*self.step + self.nheights])
1731
1732 for j in range(self.buffer.shape[0]):
1733 self.sshProfiles[j] = numpy.transpose(self.buffer[j])
1734
1735 profileIndex = self.nsamples
1736 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1737 ippSeconds = (deltaHeight*1.0e-6)/(0.15)
1738 #print("ippSeconds, dH: ",ippSeconds,deltaHeight)
1739 try:
1740 if dataOut.concat_m is not None:
1741 ippSeconds= ippSeconds/float(dataOut.concat_m)
1742 #print "Profile concat %d"%dataOut.concat_m
1743 except:
1744 pass
1745
1746 dataOut.data = self.sshProfiles
1747 dataOut.flagNoData = False
1748 dataOut.heightList = numpy.arange(self.buffer.shape[1]) *self.step*deltaHeight + dataOut.heightList[0]
1749 dataOut.nProfiles = int(dataOut.nProfiles*self.nsamples)
1750
1751 dataOut.profileIndex = profileIndex
1752 dataOut.flagDataAsBlock = True
1753 dataOut.ippSeconds = ippSeconds
1754 dataOut.step = self.step
1755 #print(numpy.shape(dataOut.data))
1756 #exit(1)
1757
1758 return dataOut
General Comments 0
You need to be logged in to leave comments. Login now