##// END OF EJS Templates
schain
RSS
Update ppi and rhi plot fix processing ppi+rhi
jespinoza -
r1562:dab94d864b0c
parent child
Show More
Side by Side Unified
Context file:
r1562:dab94d864b0c Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,723 +1,749
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """Base class to create plot operations
5 """Base class to create plot operations
6
6
7 """
7 """
8
8
9 import os
9 import os
10 import sys
10 import sys
11 import zmq
11 import zmq
12 import time
12 import time
13 import numpy
13 import numpy
14 import datetime
14 import datetime
15 from collections import deque
15 from collections import deque
16 from functools import wraps
16 from functools import wraps
17 from threading import Thread
17 from threading import Thread
18 import matplotlib,re
18 import matplotlib,re
19
19
20 if 'BACKEND' in os.environ:
20 if 'BACKEND' in os.environ:
21 matplotlib.use(os.environ['BACKEND'])
21 matplotlib.use(os.environ['BACKEND'])
22 elif 'linux' in sys.platform:
22 elif 'linux' in sys.platform:
23 matplotlib.use("TkAgg")
23 matplotlib.use("Agg")
24 elif 'darwin' in sys.platform:
24 elif 'darwin' in sys.platform:
25 matplotlib.use('MacOSX')
25 matplotlib.use('MacOSX')
26 else:
26 else:
27 from schainpy.utils import log
27 from schainpy.utils import log
28 log.warning('Using default Backend="Agg"', 'INFO')
28 log.warning('Using default Backend="Agg"', 'INFO')
29 matplotlib.use('Agg')
29 matplotlib.use('Agg')
30
30
31 import matplotlib.pyplot as plt
31 import matplotlib.pyplot as plt
32 from matplotlib.patches import Polygon
32 from matplotlib.patches import Polygon
33 from mpl_toolkits.axes_grid1 import make_axes_locatable
33 from mpl_toolkits.axes_grid1 import make_axes_locatable
34 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
34 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
35
35
36 from .plotting_codes import *
36 from .plotting_codes import *
37
37
38 from schainpy.model.data.jrodata import PlotterData
38 from schainpy.model.data.jrodata import PlotterData
39 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
39 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
40 from schainpy.utils import log
40 from schainpy.utils import log
41
41
42 for name, cb_table in sophy_cb_tables:
42 for name, cb_table in sophy_cb_tables:
43 ncmap = matplotlib.colors.ListedColormap(cb_table, name=name)
43 ncmap = matplotlib.colors.ListedColormap(cb_table, name=name)
44 matplotlib.pyplot.register_cmap(cmap=ncmap)
44 matplotlib.pyplot.register_cmap(cmap=ncmap)
45
45
46 EARTH_RADIUS = 6.3710e3
46 EARTH_RADIUS = 6.3710e3
47
47
48 def ll2xy(lat1, lon1, lat2, lon2):
48 def ll2xy(lat1, lon1, lat2, lon2):
49
49
50 p = 0.017453292519943295
50 p = 0.017453292519943295
51 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
51 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
52 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
52 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
53 r = 12742 * numpy.arcsin(numpy.sqrt(a))
53 r = 12742 * numpy.arcsin(numpy.sqrt(a))
54 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
54 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
55 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
55 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
56 theta = -theta + numpy.pi/2
56 theta = -theta + numpy.pi/2
57 return r*numpy.cos(theta), r*numpy.sin(theta)
57 return r*numpy.cos(theta), r*numpy.sin(theta)
58
58
59
59
60 def km2deg(km):
60 def km2deg(km):
61 '''
61 '''
62 Convert distance in km to degrees
62 Convert distance in km to degrees
63 '''
63 '''
64
64
65 return numpy.rad2deg(km/EARTH_RADIUS)
65 return numpy.rad2deg(km/EARTH_RADIUS)
66
66
67
67
68 def figpause(interval):
68 def figpause(interval):
69 backend = plt.rcParams['backend']
69 backend = plt.rcParams['backend']
70 if backend in matplotlib.rcsetup.interactive_bk:
70 if backend in matplotlib.rcsetup.interactive_bk:
71 figManager = matplotlib._pylab_helpers.Gcf.get_active()
71 figManager = matplotlib._pylab_helpers.Gcf.get_active()
72 if figManager is not None:
72 if figManager is not None:
73 canvas = figManager.canvas
73 canvas = figManager.canvas
74 if canvas.figure.stale:
74 if canvas.figure.stale:
75 canvas.draw()
75 canvas.draw()
76 try:
76 try:
77 canvas.start_event_loop(interval)
77 canvas.start_event_loop(interval)
78 except:
78 except:
79 pass
79 pass
80 return
80 return
81
81
82 def popup(message):
82 def popup(message):
83 '''
83 '''
84 '''
84 '''
85
85
86 fig = plt.figure(figsize=(12, 8), facecolor='r')
86 fig = plt.figure(figsize=(12, 8), facecolor='r')
87 text = '\n'.join([s.strip() for s in message.split(':')])
87 text = '\n'.join([s.strip() for s in message.split(':')])
88 fig.text(0.01, 0.5, text, ha='left', va='center',
88 fig.text(0.01, 0.5, text, ha='left', va='center',
89 size='20', weight='heavy', color='w')
89 size='20', weight='heavy', color='w')
90 fig.show()
90 fig.show()
91 figpause(1000)
91 figpause(1000)
92
92
93
93
94 class Throttle(object):
94 class Throttle(object):
95 '''
95 '''
96 Decorator that prevents a function from being called more than once every
96 Decorator that prevents a function from being called more than once every
97 time period.
97 time period.
98 To create a function that cannot be called more than once a minute, but
98 To create a function that cannot be called more than once a minute, but
99 will sleep until it can be called:
99 will sleep until it can be called:
100 @Throttle(minutes=1)
100 @Throttle(minutes=1)
101 def foo():
101 def foo():
102 pass
102 pass
103
103
104 for i in range(10):
104 for i in range(10):
105 foo()
105 foo()
106 print "This function has run %s times." % i
106 print "This function has run %s times." % i
107 '''
107 '''
108
108
109 def __init__(self, seconds=0, minutes=0, hours=0):
109 def __init__(self, seconds=0, minutes=0, hours=0):
110 self.throttle_period = datetime.timedelta(
110 self.throttle_period = datetime.timedelta(
111 seconds=seconds, minutes=minutes, hours=hours
111 seconds=seconds, minutes=minutes, hours=hours
112 )
112 )
113
113
114 self.time_of_last_call = datetime.datetime.min
114 self.time_of_last_call = datetime.datetime.min
115
115
116 def __call__(self, fn):
116 def __call__(self, fn):
117 @wraps(fn)
117 @wraps(fn)
118 def wrapper(*args, **kwargs):
118 def wrapper(*args, **kwargs):
119 coerce = kwargs.pop('coerce', None)
119 coerce = kwargs.pop('coerce', None)
120 if coerce:
120 if coerce:
121 self.time_of_last_call = datetime.datetime.now()
121 self.time_of_last_call = datetime.datetime.now()
122 return fn(*args, **kwargs)
122 return fn(*args, **kwargs)
123 else:
123 else:
124 now = datetime.datetime.now()
124 now = datetime.datetime.now()
125 time_since_last_call = now - self.time_of_last_call
125 time_since_last_call = now - self.time_of_last_call
126 time_left = self.throttle_period - time_since_last_call
126 time_left = self.throttle_period - time_since_last_call
127
127
128 if time_left > datetime.timedelta(seconds=0):
128 if time_left > datetime.timedelta(seconds=0):
129 return
129 return
130
130
131 self.time_of_last_call = datetime.datetime.now()
131 self.time_of_last_call = datetime.datetime.now()
132 return fn(*args, **kwargs)
132 return fn(*args, **kwargs)
133
133
134 return wrapper
134 return wrapper
135
135
136 def apply_throttle(value):
136 def apply_throttle(value):
137
137
138 @Throttle(seconds=value)
138 @Throttle(seconds=value)
139 def fnThrottled(fn):
139 def fnThrottled(fn):
140 fn()
140 fn()
141
141
142 return fnThrottled
142 return fnThrottled
143
143
144
144
145 @MPDecorator
145 @MPDecorator
146 class Plot(Operation):
146 class Plot(Operation):
147 """Base class for Schain plotting operations
147 """Base class for Schain plotting operations
148
148
149 This class should never be use directtly you must subclass a new operation,
149 This class should never be use directtly you must subclass a new operation,
150 children classes must be defined as follow:
150 children classes must be defined as follow:
151
151
152 ExamplePlot(Plot):
152 ExamplePlot(Plot):
153
153
154 CODE = 'code'
154 CODE = 'code'
155 colormap = 'jet'
155 colormap = 'jet'
156 plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
156 plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
157
157
158 def setup(self):
158 def setup(self):
159 pass
159 pass
160
160
161 def plot(self):
161 def plot(self):
162 pass
162 pass
163
163
164 """
164 """
165
165
166 CODE = 'Figure'
166 CODE = 'Figure'
167 colormap = 'jet'
167 colormap = 'jet'
168 bgcolor = 'white'
168 bgcolor = 'white'
169 buffering = True
169 buffering = True
170 __missing = 1E30
170 __missing = 1E30
171 projection = None
171
172
172 __attrs__ = ['show', 'save', 'ymin', 'ymax', 'zmin', 'zmax', 'title',
173 __attrs__ = ['show', 'save', 'ymin', 'ymax', 'zmin', 'zmax', 'title',
173 'showprofile']
174 'showprofile']
174
175
175 def __init__(self):
176 def __init__(self):
176
177
177 Operation.__init__(self)
178 Operation.__init__(self)
178 self.isConfig = False
179 self.isConfig = False
179 self.isPlotConfig = False
180 self.isPlotConfig = False
180 self.save_time = 0
181 self.save_time = 0
181 self.sender_time = 0
182 self.sender_time = 0
182 self.data = None
183 self.data = None
183 self.firsttime = True
184 self.firsttime = True
184 self.sender_queue = deque(maxlen=10)
185 self.sender_queue = deque(maxlen=10)
185 self.plots_adjust = {'left': 0.125, 'right': 0.9, 'bottom': 0.15, 'top': 0.9, 'wspace': 0.2, 'hspace': 0.2}
186 self.plots_adjust = {'left': 0.125, 'right': 0.9, 'bottom': 0.15, 'top': 0.9, 'wspace': 0.2, 'hspace': 0.2}
186
187
187 def __fmtTime(self, x, pos):
188 def __fmtTime(self, x, pos):
188 '''
189 '''
189 '''
190 '''
190
191
191 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
192 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
192
193
193 def __setup(self, **kwargs):
194 def __setup(self, **kwargs):
194 '''
195 '''
195 Initialize variables
196 Initialize variables
196 '''
197 '''
197
198
198 self.figures = []
199 self.figures = []
199 self.axes = []
200 self.axes = []
200 self.cb_axes = []
201 self.cb_axes = []
201 self.localtime = kwargs.pop('localtime', True)
202 self.localtime = kwargs.pop('localtime', True)
202 self.show = kwargs.get('show', True)
203 self.show = kwargs.get('show', True)
203 self.save = kwargs.get('save', False)
204 self.save = kwargs.get('save', False)
204 self.save_period = kwargs.get('save_period', 0)
205 self.save_period = kwargs.get('save_period', 0)
205 self.colormap = kwargs.get('colormap', self.colormap)
206 self.colormap = kwargs.get('colormap', self.colormap)
206 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
207 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
207 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
208 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
208 self.colormaps = kwargs.get('colormaps', None)
209 self.colormaps = kwargs.get('colormaps', None)
209 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
210 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
210 self.showprofile = kwargs.get('showprofile', False)
211 self.showprofile = kwargs.get('showprofile', False)
211 self.title = kwargs.get('wintitle', self.CODE.upper())
212 self.title = kwargs.get('wintitle', self.CODE.upper())
212 self.cb_label = kwargs.get('cb_label', None)
213 self.cb_label = kwargs.get('cb_label', None)
213 self.cb_labels = kwargs.get('cb_labels', None)
214 self.cb_labels = kwargs.get('cb_labels', None)
214 self.labels = kwargs.get('labels', None)
215 self.labels = kwargs.get('labels', None)
215 self.xaxis = kwargs.get('xaxis', 'frequency')
216 self.xaxis = kwargs.get('xaxis', 'frequency')
216 self.zmin = kwargs.get('zmin', None)
217 self.zmin = kwargs.get('zmin', None)
217 self.zmax = kwargs.get('zmax', None)
218 self.zmax = kwargs.get('zmax', None)
218 self.zlimits = kwargs.get('zlimits', None)
219 self.zlimits = kwargs.get('zlimits', None)
219 self.xmin = kwargs.get('xmin', None)
220 self.xmin = kwargs.get('xmin', None)
220 self.xmax = kwargs.get('xmax', None)
221 self.xmax = kwargs.get('xmax', None)
221 self.xrange = kwargs.get('xrange', 12)
222 self.xrange = kwargs.get('xrange', 12)
222 self.xscale = kwargs.get('xscale', None)
223 self.xscale = kwargs.get('xscale', None)
223 self.ymin = kwargs.get('ymin', None)
224 self.ymin = kwargs.get('ymin', None)
224 self.ymax = kwargs.get('ymax', None)
225 self.ymax = kwargs.get('ymax', None)
225 self.yscale = kwargs.get('yscale', None)
226 self.yscale = kwargs.get('yscale', None)
226 self.xlabel = kwargs.get('xlabel', None)
227 self.xlabel = kwargs.get('xlabel', None)
227 self.attr_time = kwargs.get('attr_time', 'utctime')
228 self.attr_time = kwargs.get('attr_time', 'utctime')
228 self.attr_data = kwargs.get('attr_data', 'data_param')
229 self.attr_data = kwargs.get('attr_data', 'data_param')
229 self.decimation = kwargs.get('decimation', None)
230 self.decimation = kwargs.get('decimation', None)
230 self.oneFigure = kwargs.get('oneFigure', True)
231 self.oneFigure = kwargs.get('oneFigure', True)
231 self.width = kwargs.get('width', None)
232 self.width = kwargs.get('width', None)
232 self.height = kwargs.get('height', None)
233 self.height = kwargs.get('height', None)
233 self.colorbar = kwargs.get('colorbar', True)
234 self.colorbar = kwargs.get('colorbar', True)
234 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
235 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
235 self.channels = kwargs.get('channels', None)
236 self.channels = kwargs.get('channels', None)
236 self.titles = kwargs.get('titles', [])
237 self.titles = kwargs.get('titles', [])
237 self.polar = False
238 self.polar = False
238 self.type = kwargs.get('type', 'iq')
239 self.type = kwargs.get('type', 'iq')
239 self.grid = kwargs.get('grid', False)
240 self.grid = kwargs.get('grid', False)
240 self.pause = kwargs.get('pause', False)
241 self.pause = kwargs.get('pause', False)
241 self.save_code = kwargs.get('save_code', self.CODE)
242 self.save_code = kwargs.get('save_code', self.CODE)
242 self.throttle = kwargs.get('throttle', 0)
243 self.throttle = kwargs.get('throttle', 0)
243 self.exp_code = kwargs.get('exp_code', None)
244 self.exp_code = kwargs.get('exp_code', None)
244 self.server = kwargs.get('server', False)
245 self.server = kwargs.get('server', False)
245 self.sender_period = kwargs.get('sender_period', 60)
246 self.sender_period = kwargs.get('sender_period', 60)
246 self.tag = kwargs.get('tag', '')
247 self.tag = kwargs.get('tag', '')
247 self.height_index = kwargs.get('height_index', None)
248 self.height_index = kwargs.get('height_index', None)
248 self.__throttle_plot = apply_throttle(self.throttle)
249 self.__throttle_plot = apply_throttle(self.throttle)
249 code = self.attr_data if self.attr_data else self.CODE
250 code = self.attr_data if self.attr_data else self.CODE
250 self.data = PlotterData(self.CODE, self.exp_code, self.localtime)
251 self.data = PlotterData(self.CODE, self.exp_code, self.localtime)
251 self.ang_min = kwargs.get('ang_min', None)
252 self.ang_min = kwargs.get('ang_min', None)
252 self.ang_max = kwargs.get('ang_max', None)
253 self.ang_max = kwargs.get('ang_max', None)
253 self.mode = kwargs.get('mode', None)
254 self.mode = kwargs.get('mode', None)
254 self.mask = kwargs.get('mask', False)
255 self.mask = kwargs.get('mask', False)
255
256 self.shapes = kwargs.get('shapes', './')
256
257
257 if self.server:
258 if self.server:
258 if not self.server.startswith('tcp://'):
259 if not self.server.startswith('tcp://'):
259 self.server = 'tcp://{}'.format(self.server)
260 self.server = 'tcp://{}'.format(self.server)
260 log.success(
261 log.success(
261 'Sending to server: {}'.format(self.server),
262 'Sending to server: {}'.format(self.server),
262 self.name
263 self.name
263 )
264 )
264
265
265 if isinstance(self.attr_data, str):
266 if isinstance(self.attr_data, str):
266 self.attr_data = [self.attr_data]
267 self.attr_data = [self.attr_data]
267
268
268 def __setup_plot(self):
269 def __setup_plot(self):
269 '''
270 '''
270 Common setup for all figures, here figures and axes are created
271 Common setup for all figures, here figures and axes are created
271 '''
272 '''
272
273
273 self.setup()
274 self.setup()
274
275
275 self.time_label = 'LT' if self.localtime else 'UTC'
276 self.time_label = 'LT' if self.localtime else 'UTC'
276
277
277 if self.width is None:
278 if self.width is None:
278 self.width = 8
279 self.width = 8
279
280
280 self.figures = []
281 self.figures = {'PPI':[], 'RHI':[]}
281 self.axes = []
282 self.axes = {'PPI':[], 'RHI':[]}
282 self.cb_axes = []
283 self.cb_axes = []
283 self.pf_axes = []
284 self.pf_axes = []
284 self.cmaps = []
285 self.cmaps = []
285
286
286 size = '15%' if self.ncols == 1 else '30%'
287 size = '15%' if self.ncols == 1 else '30%'
287 pad = '4%' if self.ncols == 1 else '8%'
288 pad = '4%' if self.ncols == 1 else '8%'
288
289
289 if self.oneFigure:
290 if self.oneFigure:
290 if self.height is None:
291 if self.height is None:
291 self.height = 1.4 * self.nrows + 1
292 self.height = 1.4 * self.nrows + 1
292 fig = plt.figure(figsize=(self.width, self.height),
293 fig_p = plt.figure(figsize=(self.width, self.height),
293 edgecolor='k',
294 edgecolor='k',
294 facecolor='w')
295 facecolor='w')
295 self.figures.append(fig)
296 fig_r = plt.figure(figsize=(self.width, self.height),
296 for n in range(self.nplots):
297 edgecolor='k',
297 ax = fig.add_subplot(self.nrows, self.ncols,
298 facecolor='w')
298 n + 1, polar=self.polar)
299 self.figures['PPI'].append(fig_p)
299 ax.tick_params(labelsize=8)
300 self.figures['RHI'].append(fig_r)
300 ax.firsttime = True
301 for n in range(self.nplots):
301 ax.index = 0
302 ax_p = fig_p.add_subplot(self.nrows, self.ncols, n+1, polar=self.polar, projection=self.projection)
302 ax.press = None
303 ax_r = fig_r.add_subplot(self.nrows, self.ncols, n+1, polar=self.polar)
303 self.axes.append(ax)
304 ax_p.tick_params(labelsize=8)
305 ax_p.firsttime = True
306 ax_p.index = 0
307 ax_p.press = None
308 ax_r.tick_params(labelsize=8)
309 ax_r.firsttime = True
310 ax_r.index = 0
311 ax_r.press = None
312
313 self.axes['PPI'].append(ax_p)
314 self.axes['RHI'].append(ax_r)
315
304 if self.showprofile:
316 if self.showprofile:
305 cax = self.__add_axes(ax, size=size, pad=pad)
317 cax = self.__add_axes(ax, size=size, pad=pad)
306 cax.tick_params(labelsize=8)
318 cax.tick_params(labelsize=8)
307 self.pf_axes.append(cax)
319 self.pf_axes.append(cax)
308 else:
320 else:
309 if self.height is None:
321 if self.height is None:
310 self.height = 3
322 self.height = 3
311 for n in range(self.nplots):
323 for n in range(self.nplots):
312 fig = plt.figure(figsize=(self.width, self.height),
324 fig = plt.figure(figsize=(self.width, self.height),
313 edgecolor='k',
325 edgecolor='k',
314 facecolor='w')
326 facecolor='w')
315 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
327 ax_p = fig.add_subplot(1, 1, 1, polar=self.polar, projection=self.projection)
316 ax.tick_params(labelsize=8)
328 ax_r = fig.add_subplot(1, 1, 1, polar=self.polar)
317 ax.firsttime = True
329 ax_p.tick_params(labelsize=8)
318 ax.index = 0
330 ax_p.firsttime = True
319 ax.press = None
331 ax_p.index = 0
332 ax_p.press = None
333 ax_r.tick_params(labelsize=8)
334 ax_r.firsttime = True
335 ax_r.index = 0
336 ax_r.press = None
320 self.figures.append(fig)
337 self.figures.append(fig)
321 self.axes.append(ax)
338 self.axes['PPI'].append(ax_p)
339 self.axes['RHI'].append(ax_r)
322 if self.showprofile:
340 if self.showprofile:
323 cax = self.__add_axes(ax, size=size, pad=pad)
341 cax = self.__add_axes(ax, size=size, pad=pad)
324 cax.tick_params(labelsize=8)
342 cax.tick_params(labelsize=8)
325 self.pf_axes.append(cax)
343 self.pf_axes.append(cax)
326
344
327 for n in range(self.nrows):
345 for n in range(self.nrows):
328 if self.colormaps is not None:
346 if self.colormaps is not None:
329 cmap = plt.get_cmap(self.colormaps[n])
347 cmap = plt.get_cmap(self.colormaps[n])
330 else:
348 else:
331 cmap = plt.get_cmap(self.colormap)
349 cmap = plt.get_cmap(self.colormap)
332 cmap.set_bad(self.bgcolor, 1.)
350 cmap.set_bad(self.bgcolor, 1.)
333 self.cmaps.append(cmap)
351 self.cmaps.append(cmap)
334
352
335 def __add_axes(self, ax, size='30%', pad='8%'):
353 def __add_axes(self, ax, size='30%', pad='8%'):
336 '''
354 '''
337 Add new axes to the given figure
355 Add new axes to the given figure
338 '''
356 '''
339 divider = make_axes_locatable(ax)
357 divider = make_axes_locatable(ax)
340 nax = divider.new_horizontal(size=size, pad=pad)
358 nax = divider.new_horizontal(size=size, pad=pad)
341 ax.figure.add_axes(nax)
359 ax.figure.add_axes(nax)
342 return nax
360 return nax
343
361
344 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
362 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
345 '''
363 '''
346 Create a masked array for missing data
364 Create a masked array for missing data
347 '''
365 '''
348 if x_buffer.shape[0] < 2:
366 if x_buffer.shape[0] < 2:
349 return x_buffer, y_buffer, z_buffer
367 return x_buffer, y_buffer, z_buffer
350
368
351 deltas = x_buffer[1:] - x_buffer[0:-1]
369 deltas = x_buffer[1:] - x_buffer[0:-1]
352 x_median = numpy.median(deltas)
370 x_median = numpy.median(deltas)
353
371
354 index = numpy.where(deltas > 5 * x_median)
372 index = numpy.where(deltas > 5 * x_median)
355
373
356 if len(index[0]) != 0:
374 if len(index[0]) != 0:
357 z_buffer[::, index[0], ::] = self.__missing
375 z_buffer[::, index[0], ::] = self.__missing
358 z_buffer = numpy.ma.masked_inside(z_buffer,
376 z_buffer = numpy.ma.masked_inside(z_buffer,
359 0.99 * self.__missing,
377 0.99 * self.__missing,
360 1.01 * self.__missing)
378 1.01 * self.__missing)
361
379
362 return x_buffer, y_buffer, z_buffer
380 return x_buffer, y_buffer, z_buffer
363
381
364 def decimate(self):
382 def decimate(self):
365
383
366 # dx = int(len(self.x)/self.__MAXNUMX) + 1
384 # dx = int(len(self.x)/self.__MAXNUMX) + 1
367 dy = int(len(self.y) / self.decimation) + 1
385 dy = int(len(self.y) / self.decimation) + 1
368
386
369 # x = self.x[::dx]
387 # x = self.x[::dx]
370 x = self.x
388 x = self.x
371 y = self.y[::dy]
389 y = self.y[::dy]
372 z = self.z[::, ::, ::dy]
390 z = self.z[::, ::, ::dy]
373
391
374 return x, y, z
392 return x, y, z
375
393
376 def format(self):
394 def format(self):
377 '''
395 '''
378 Set min and max values, labels, ticks and titles
396 Set min and max values, labels, ticks and titles
379 '''
397 '''
380
398
381 for n, ax in enumerate(self.axes):
399 for n, ax in enumerate(self.axes[self.mode]):
382 if ax.firsttime:
400 if ax.firsttime:
383 if self.xaxis != 'time':
401 if self.xaxis != 'time':
384 xmin = self.xmin
402 xmin = self.xmin
385 xmax = self.xmax
403 xmax = self.xmax
386 else:
404 else:
387 xmin = self.tmin
405 xmin = self.tmin
388 xmax = self.tmin + self.xrange*60*60
406 xmax = self.tmin + self.xrange*60*60
389 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
407 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
390 ax.xaxis.set_major_locator(LinearLocator(9))
408 ax.xaxis.set_major_locator(LinearLocator(9))
391 ymin = self.ymin if self.ymin is not None else numpy.nanmin(self.y[numpy.isfinite(self.y)])
409 ymin = self.ymin if self.ymin is not None else numpy.nanmin(self.y[numpy.isfinite(self.y)])
392 ymax = self.ymax if self.ymax is not None else numpy.nanmax(self.y[numpy.isfinite(self.y)])
410 ymax = self.ymax if self.ymax is not None else numpy.nanmax(self.y[numpy.isfinite(self.y)])
411
393 ax.set_facecolor(self.bgcolor)
412 ax.set_facecolor(self.bgcolor)
413
394 if self.xscale:
414 if self.xscale:
395 ax.xaxis.set_major_formatter(FuncFormatter(
415 ax.xaxis.set_major_formatter(FuncFormatter(
396 lambda x, pos: '{0:g}'.format(x*self.xscale)))
416 lambda x, pos: '{0:g}'.format(x*self.xscale)))
397 if self.yscale:
417 if self.yscale:
398 ax.yaxis.set_major_formatter(FuncFormatter(
418 ax.yaxis.set_major_formatter(FuncFormatter(
399 lambda x, pos: '{0:g}'.format(x*self.yscale)))
419 lambda x, pos: '{0:g}'.format(x*self.yscale)))
400 if self.xlabel is not None:
420 if self.xlabel is not None:
401 ax.set_xlabel(self.xlabel)
421 ax.set_xlabel(self.xlabel)
402 if self.ylabel is not None:
422 if self.ylabel is not None:
403 ax.set_ylabel(self.ylabel)
423 ax.set_ylabel(self.ylabel)
404 if self.showprofile:
424 if self.showprofile:
405 self.pf_axes[n].set_ylim(ymin, ymax)
425 self.pf_axes[n].set_ylim(ymin, ymax)
406 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
426 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
407 self.pf_axes[n].set_xlabel('dB')
427 self.pf_axes[n].set_xlabel('dB')
408 self.pf_axes[n].grid(b=True, axis='x')
428 self.pf_axes[n].grid(b=True, axis='x')
409 [tick.set_visible(False)
429 [tick.set_visible(False)
410 for tick in self.pf_axes[n].get_yticklabels()]
430 for tick in self.pf_axes[n].get_yticklabels()]
411 if self.colorbar:
431 if self.colorbar:
412 ax.cbar = plt.colorbar(
432 ax.cbar = plt.colorbar(
413 ax.plt, ax=ax, fraction=0.05, pad=0.06, aspect=10)
433 ax.plt, ax=ax, fraction=0.05, pad=0.06, aspect=10)
414 ax.cbar.ax.tick_params(labelsize=8)
434 ax.cbar.ax.tick_params(labelsize=8)
415 ax.cbar.ax.press = None
435 ax.cbar.ax.press = None
416 if self.cb_label:
436 if self.cb_label:
417 ax.cbar.set_label(self.cb_label, size=8)
437 ax.cbar.set_label(self.cb_label, size=8)
418 elif self.cb_labels:
438 elif self.cb_labels:
419 ax.cbar.set_label(self.cb_labels[n], size=8)
439 ax.cbar.set_label(self.cb_labels[n], size=8)
420 else:
440 else:
421 ax.cbar = None
441 ax.cbar = None
422 ax.set_xlim(xmin, xmax)
442 if self.mode == 'RHI':
423 ax.set_ylim(ymin, ymax)
443 ax.set_xlim(xmin, xmax)
444 ax.set_ylim(ymin, ymax)
424 ax.firsttime = False
445 ax.firsttime = False
425 if self.grid:
446 if self.grid:
426 ax.grid(True)
447 ax.grid(True)
427 if not self.polar:
448 if not self.polar:
428 ax.set_title('{} {} {}'.format(
449 ax.set_title('{} {} {}'.format(
429 self.titles[n],
450 self.titles[n],
430 self.getDateTime(self.data.max_time).strftime(
451 self.getDateTime(self.data.max_time).strftime(
431 '%Y-%m-%d %H:%M:%S'),
452 '%Y-%m-%d %H:%M:%S'),
432 self.time_label),
453 self.time_label),
433 size=8)
454 size=8)
434 else:
455 else:
435 #ax.set_title('{}'.format(self.titles[n]), size=8)
456 #ax.set_title('{}'.format(self.titles[n]), size=8)
436 ax.set_title('{} {} {}'.format(
457 ax.set_title('{} {} {}'.format(
437 self.titles[n],
458 self.titles[n],
438 self.getDateTime(self.data.max_time).strftime(
459 self.getDateTime(self.data.max_time).strftime(
439 '%Y-%m-%d %H:%M:%S'),
460 '%Y-%m-%d %H:%M:%S'),
440 self.time_label),
461 self.time_label),
441 size=8)
462 size=8)
442 ax.set_ylim(0, self.ymax)
463 ax.set_ylim(0, self.ymax)
443 if self.mode == 'PPI':
464 if self.mode == 'PPI':
444 ax.set_yticks(ax.get_yticks(), labels=ax.get_yticks(), color='white')
465 ax.set_yticks(ax.get_yticks(), labels=ax.get_yticks(), color='white')
445 ax.yaxis.labelpad = 28
466 ax.yaxis.labelpad = 28
446 elif self.mode == 'RHI':
467 elif self.mode == 'RHI':
447 ax.xaxis.labelpad = 16
468 ax.xaxis.labelpad = 16
448
469
449 if self.firsttime:
470 if self.firsttime:
450 for n, fig in enumerate(self.figures):
471 for fig in self.figures['PPI'] + self.figures['RHI']:
451 fig.subplots_adjust(**self.plots_adjust)
472 fig.subplots_adjust(**self.plots_adjust)
452 self.firsttime = False
473 self.firsttime = False
453
474
454 def clear_figures(self):
475 def clear_figures(self):
455 '''
476 '''
456 Reset axes for redraw plots
477 Reset axes for redraw plots
457 '''
478 '''
458
479
459 for ax in self.axes+self.pf_axes+self.cb_axes:
480 axes = self.pf_axes + self.cb_axes + self.axes[self.mode]
481
482 for ax in axes:
460 ax.clear()
483 ax.clear()
461 ax.firsttime = True
484 ax.firsttime = True
462 if hasattr(ax, 'cbar') and ax.cbar:
485 if hasattr(ax, 'cbar') and ax.cbar:
463 ax.cbar.remove()
486 ax.cbar.remove()
464
487
465 def __plot(self):
488 def __plot(self):
466 '''
489 '''
467 Main function to plot, format and save figures
490 Main function to plot, format and save figures
468 '''
491 '''
469
492
470 self.plot()
493 self.plot()
471 self.format()
494 self.format()
472
495 figures = self.figures[self.mode]
473 for n, fig in enumerate(self.figures):
496 for n, fig in enumerate(figures):
474 if self.nrows == 0 or self.nplots == 0:
497 if self.nrows == 0 or self.nplots == 0:
475 log.warning('No data', self.name)
498 log.warning('No data', self.name)
476 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
499 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
477 fig.canvas.manager.set_window_title(self.CODE)
500 fig.canvas.manager.set_window_title(self.CODE)
478 continue
501 continue
479
502
480 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
503 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
481 self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
504 self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
482 fig.canvas.draw()
505 fig.canvas.draw()
483 if self.show:
506 if self.show:
484 fig.show()
507 fig.show()
485 figpause(0.01)
508 figpause(0.01)
486
509
487 if self.save:
510 if self.save:
488 self.save_figure(n)
511 self.save_figure(n)
489
512
490 if self.server:
513 if self.server:
491 if self.mode and self.mode == 'RHI':
514 if self.mode and self.mode == 'RHI':
492 return
515 return
493 self.send_to_server()
516 self.send_to_server()
494
517
495 def __update(self, dataOut, timestamp):
518 def __update(self, dataOut, timestamp):
496 '''
519 '''
497 '''
520 '''
498
521
499 metadata = {
522 metadata = {
500 'yrange': dataOut.heightList,
523 'yrange': dataOut.heightList,
501 'interval': dataOut.timeInterval,
524 'interval': dataOut.timeInterval,
502 'channels': dataOut.channelList
525 'channels': dataOut.channelList
503 }
526 }
504
527
505 data, meta = self.update(dataOut)
528 data, meta = self.update(dataOut)
506 metadata.update(meta)
529 metadata.update(meta)
507 self.data.update(data, timestamp, metadata)
530 self.data.update(data, timestamp, metadata)
508
531
509 def save_figure(self, n):
532 def save_figure(self, n):
510 '''
533 '''
511 '''
534 '''
512 if self.mode is not None:
535 if self.mode is not None:
513 ang = 'AZ' if self.mode == 'RHI' else 'EL'
536 ang = 'AZ' if self.mode == 'RHI' else 'EL'
514 label = '_{}_{}_{}'.format(self.mode, ang, self.mode_value)
537 folder = '_{}_{}_{}'.format(self.mode, ang, self.mode_value)
538 label = '{}{}_{}'.format(ang[0], self.mode_value, self.save_code)
515 else:
539 else:
540 folder = ''
516 label = ''
541 label = ''
517
542
518 if self.oneFigure:
543 if self.oneFigure:
519 if (self.data.max_time - self.save_time) <= self.save_period:
544 if (self.data.max_time - self.save_time) <= self.save_period:
520 return
545 return
521
546
522 self.save_time = self.data.max_time
547 self.save_time = self.data.max_time
523
548
524 fig = self.figures[n]
549 fig = self.figures[self.mode][n]
525
550
526 if self.throttle == 0:
551 if self.throttle == 0:
527 if self.oneFigure:
552 if self.oneFigure:
528 figname = os.path.join(
553 figname = os.path.join(
529 self.save,
554 self.save,
530 self.save_code + label,
555 self.save_code + folder,
531 '{}_{}.png'.format(
556 '{}_{}_{}.png'.format(
532 self.save_code + label,
557 'SOPHY',
533 self.getDateTime(self.data.max_time).strftime(
558 self.getDateTime(self.data.max_time).strftime(
534 '%Y%m%d_%H%M%S'
559 '%Y%m%d_%H%M%S'
535 ),
560 ),
561 label
536 )
562 )
537 )
563 )
538 else:
564 else:
539 figname = os.path.join(
565 figname = os.path.join(
540 self.save,
566 self.save,
541 self.save_code,
567 self.save_code,
542 '{}_ch{}_{}.png'.format(
568 '{}_ch{}_{}.png'.format(
543 self.save_code, n,
569 self.save_code, n,
544 self.getDateTime(self.data.max_time).strftime(
570 self.getDateTime(self.data.max_time).strftime(
545 '%Y%m%d_%H%M%S'
571 '%Y%m%d_%H%M%S'
546 ),
572 ),
547 )
573 )
548 )
574 )
549 log.log('Saving figure: {}'.format(figname), self.name)
575 log.log('Saving figure: {}'.format(figname), self.name)
550 if not os.path.isdir(os.path.dirname(figname)):
576 if not os.path.isdir(os.path.dirname(figname)):
551 os.makedirs(os.path.dirname(figname))
577 os.makedirs(os.path.dirname(figname))
552 fig.savefig(figname)
578 fig.savefig(figname)
553
579
554 figname = os.path.join(
580 figname = os.path.join(
555 self.save,
581 self.save,
556 '{}_{}.png'.format(
582 '{}_{}.png'.format(
557 self.save_code,
583 self.save_code,
558 self.getDateTime(self.data.min_time).strftime(
584 self.getDateTime(self.data.min_time).strftime(
559 '%Y%m%d'
585 '%Y%m%d'
560 ),
586 ),
561 )
587 )
562 )
588 )
563
589
564 log.log('Saving figure: {}'.format(figname), self.name)
590 log.log('Saving figure: {}'.format(figname), self.name)
565 if not os.path.isdir(os.path.dirname(figname)):
591 if not os.path.isdir(os.path.dirname(figname)):
566 os.makedirs(os.path.dirname(figname))
592 os.makedirs(os.path.dirname(figname))
567 fig.savefig(figname)
593 fig.savefig(figname)
568
594
569 def send_to_server(self):
595 def send_to_server(self):
570 '''
596 '''
571 '''
597 '''
572
598
573 if self.exp_code == None:
599 if self.exp_code == None:
574 log.warning('Missing `exp_code` skipping sending to server...')
600 log.warning('Missing `exp_code` skipping sending to server...')
575
601
576 last_time = self.data.max_time
602 last_time = self.data.max_time
577 interval = last_time - self.sender_time
603 interval = last_time - self.sender_time
578 if interval < self.sender_period:
604 if interval < self.sender_period:
579 return
605 return
580
606
581 self.sender_time = last_time
607 self.sender_time = last_time
582
608
583 attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax']
609 attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax']
584 for attr in attrs:
610 for attr in attrs:
585 value = getattr(self, attr)
611 value = getattr(self, attr)
586 if value:
612 if value:
587 if isinstance(value, (numpy.float32, numpy.float64)):
613 if isinstance(value, (numpy.float32, numpy.float64)):
588 value = round(float(value), 2)
614 value = round(float(value), 2)
589 self.data.meta[attr] = value
615 self.data.meta[attr] = value
590 if self.colormap == 'jet' or self.colormap == 'sophy_w':
616 if self.colormap == 'jet' or self.colormap == 'sophy_w':
591 self.data.meta['colormap'] = 'Jet'
617 self.data.meta['colormap'] = 'Jet'
592 elif 'sophy_v' in self.colormap:
618 elif 'sophy_v' in self.colormap:
593 self.data.meta['colormap'] = 'RdBu'
619 self.data.meta['colormap'] = 'RdBu'
594 else:
620 else:
595 self.data.meta['colormap'] = 'Viridis'
621 self.data.meta['colormap'] = 'Viridis'
596 self.data.meta['interval'] = int(interval)
622 self.data.meta['interval'] = int(interval)
597
623
598 self.sender_queue.append(last_time)
624 self.sender_queue.append(last_time)
599
625
600 while True:
626 while True:
601 try:
627 try:
602 tm = self.sender_queue.popleft()
628 tm = self.sender_queue.popleft()
603 except IndexError:
629 except IndexError:
604 break
630 break
605 msg = self.data.jsonify(tm, self.save_code, self.plot_type, key='var')
631 msg = self.data.jsonify(tm, self.save_code, self.plot_type, key='var')
606 self.socket.send_string(msg)
632 self.socket.send_string(msg)
607 socks = dict(self.poll.poll(2000))
633 socks = dict(self.poll.poll(2000))
608 if socks.get(self.socket) == zmq.POLLIN:
634 if socks.get(self.socket) == zmq.POLLIN:
609 reply = self.socket.recv_string()
635 reply = self.socket.recv_string()
610 if reply == 'ok':
636 if reply == 'ok':
611 log.log("Response from server ok", self.name)
637 log.log("Response from server ok", self.name)
612 time.sleep(0.1)
638 time.sleep(0.1)
613 continue
639 continue
614 else:
640 else:
615 log.warning(
641 log.warning(
616 "Malformed reply from server: {}".format(reply), self.name)
642 "Malformed reply from server: {}".format(reply), self.name)
617 else:
643 else:
618 log.warning(
644 log.warning(
619 "No response from server, retrying...", self.name)
645 "No response from server, retrying...", self.name)
620 self.sender_queue.appendleft(tm)
646 self.sender_queue.appendleft(tm)
621 self.socket.setsockopt(zmq.LINGER, 0)
647 self.socket.setsockopt(zmq.LINGER, 0)
622 self.socket.close()
648 self.socket.close()
623 self.poll.unregister(self.socket)
649 self.poll.unregister(self.socket)
624 self.socket = self.context.socket(zmq.REQ)
650 self.socket = self.context.socket(zmq.REQ)
625 self.socket.connect(self.server)
651 self.socket.connect(self.server)
626 self.poll.register(self.socket, zmq.POLLIN)
652 self.poll.register(self.socket, zmq.POLLIN)
627 break
653 break
628
654
629 def setup(self):
655 def setup(self):
630 '''
656 '''
631 This method should be implemented in the child class, the following
657 This method should be implemented in the child class, the following
632 attributes should be set:
658 attributes should be set:
633
659
634 self.nrows: number of rows
660 self.nrows: number of rows
635 self.ncols: number of cols
661 self.ncols: number of cols
636 self.nplots: number of plots (channels or pairs)
662 self.nplots: number of plots (channels or pairs)
637 self.ylabel: label for Y axes
663 self.ylabel: label for Y axes
638 self.titles: list of axes title
664 self.titles: list of axes title
639
665
640 '''
666 '''
641 raise NotImplementedError
667 raise NotImplementedError
642
668
643 def plot(self):
669 def plot(self):
644 '''
670 '''
645 Must be defined in the child class, the actual plotting method
671 Must be defined in the child class, the actual plotting method
646 '''
672 '''
647 raise NotImplementedError
673 raise NotImplementedError
648
674
649 def update(self, dataOut):
675 def update(self, dataOut):
650 '''
676 '''
651 Must be defined in the child class, update self.data with new data
677 Must be defined in the child class, update self.data with new data
652 '''
678 '''
653
679
654 data = {
680 data = {
655 self.CODE: getattr(dataOut, 'data_{}'.format(self.CODE))
681 self.CODE: getattr(dataOut, 'data_{}'.format(self.CODE))
656 }
682 }
657 meta = {}
683 meta = {}
658
684
659 return data, meta
685 return data, meta
660
686
661 def run(self, dataOut, **kwargs):
687 def run(self, dataOut, **kwargs):
662 '''
688 '''
663 Main plotting routine
689 Main plotting routine
664 '''
690 '''
665
691
666 if self.isConfig is False:
692 if self.isConfig is False:
667 self.__setup(**kwargs)
693 self.__setup(**kwargs)
668
694
669 if self.localtime:
695 if self.localtime:
670 self.getDateTime = datetime.datetime.fromtimestamp
696 self.getDateTime = datetime.datetime.fromtimestamp
671 else:
697 else:
672 self.getDateTime = datetime.datetime.utcfromtimestamp
698 self.getDateTime = datetime.datetime.utcfromtimestamp
673
699
674 self.data.setup()
700 self.data.setup()
675 self.isConfig = True
701 self.isConfig = True
676 if self.server:
702 if self.server:
677 self.context = zmq.Context()
703 self.context = zmq.Context()
678 self.socket = self.context.socket(zmq.REQ)
704 self.socket = self.context.socket(zmq.REQ)
679 self.socket.connect(self.server)
705 self.socket.connect(self.server)
680 self.poll = zmq.Poller()
706 self.poll = zmq.Poller()
681 self.poll.register(self.socket, zmq.POLLIN)
707 self.poll.register(self.socket, zmq.POLLIN)
682
708
683 tm = getattr(dataOut, self.attr_time)
709 tm = getattr(dataOut, self.attr_time)
684
710
685 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
711 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
686 self.save_time = tm
712 self.save_time = tm
687 self.__plot()
713 self.__plot()
688 self.tmin += self.xrange*60*60
714 self.tmin += self.xrange*60*60
689 self.data.setup()
715 self.data.setup()
690 self.clear_figures()
716 self.clear_figures()
691
717
692 self.__update(dataOut, tm)
718 self.__update(dataOut, tm)
693
719
694 if self.isPlotConfig is False:
720 if self.isPlotConfig is False:
695 self.__setup_plot()
721 self.__setup_plot()
696 self.isPlotConfig = True
722 self.isPlotConfig = True
697 if self.xaxis == 'time':
723 if self.xaxis == 'time':
698 dt = self.getDateTime(tm)
724 dt = self.getDateTime(tm)
699 if self.xmin is None:
725 if self.xmin is None:
700 self.tmin = tm
726 self.tmin = tm
701 self.xmin = dt.hour
727 self.xmin = dt.hour
702 minutes = (self.xmin-int(self.xmin)) * 60
728 minutes = (self.xmin-int(self.xmin)) * 60
703 seconds = (minutes - int(minutes)) * 60
729 seconds = (minutes - int(minutes)) * 60
704 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
730 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
705 datetime.datetime(1970, 1, 1)).total_seconds()
731 datetime.datetime(1970, 1, 1)).total_seconds()
706 if self.localtime:
732 if self.localtime:
707 self.tmin += time.timezone
733 self.tmin += time.timezone
708
734
709 if self.xmin is not None and self.xmax is not None:
735 if self.xmin is not None and self.xmax is not None:
710 self.xrange = self.xmax - self.xmin
736 self.xrange = self.xmax - self.xmin
711
737
712 if self.throttle == 0:
738 if self.throttle == 0:
713 self.__plot()
739 self.__plot()
714 else:
740 else:
715 self.__throttle_plot(self.__plot)#, coerce=coerce)
741 self.__throttle_plot(self.__plot)#, coerce=coerce)
716
742
717 def close(self):
743 def close(self):
718
744
719 if self.data and not self.data.flagNoData:
745 if self.data and not self.data.flagNoData:
720 self.save_time = 0
746 self.save_time = 0
721 self.__plot()
747 self.__plot()
722 if self.data and not self.data.flagNoData and self.pause:
748 if self.data and not self.data.flagNoData and self.pause:
723 figpause(10)
749 figpause(10)
Show file before | Show file after | Hide comments
@@ -1,697 +1,716
1 import os
1 import os
2 import datetime
2 import datetime
3 import warnings
3 import warnings
4 import numpy
4 import numpy
5 from mpl_toolkits.axisartist.grid_finder import FixedLocator, DictFormatter
5 from mpl_toolkits.axisartist.grid_finder import FixedLocator, DictFormatter
6 from matplotlib.patches import Circle
7 import cartopy.crs as ccrs
8 from cartopy.feature import ShapelyFeature
9 import cartopy.io.shapereader as shpreader
6
10
7 from schainpy.model.graphics.jroplot_base import Plot, plt
11 from schainpy.model.graphics.jroplot_base import Plot, plt
8 from schainpy.model.graphics.jroplot_spectra import SpectraPlot, RTIPlot, CoherencePlot, SpectraCutPlot
12 from schainpy.model.graphics.jroplot_spectra import SpectraPlot, RTIPlot, CoherencePlot, SpectraCutPlot
9 from schainpy.utils import log
13 from schainpy.utils import log
10
14
11
15
12 EARTH_RADIUS = 6.3710e3
16 EARTH_RADIUS = 6.3710e3
13
17
14
18
15 def antenna_to_cartesian(ranges, azimuths, elevations):
19 def antenna_to_cartesian(ranges, azimuths, elevations):
16 """
20 """
17 Return Cartesian coordinates from antenna coordinates.
21 Return Cartesian coordinates from antenna coordinates.
18
22
19 Parameters
23 Parameters
20 ----------
24 ----------
21 ranges : array
25 ranges : array
22 Distances to the center of the radar gates (bins) in kilometers.
26 Distances to the center of the radar gates (bins) in kilometers.
23 azimuths : array
27 azimuths : array
24 Azimuth angle of the radar in degrees.
28 Azimuth angle of the radar in degrees.
25 elevations : array
29 elevations : array
26 Elevation angle of the radar in degrees.
30 Elevation angle of the radar in degrees.
27
31
28 Returns
32 Returns
29 -------
33 -------
30 x, y, z : array
34 x, y, z : array
31 Cartesian coordinates in meters from the radar.
35 Cartesian coordinates in meters from the radar.
32
36
33 Notes
37 Notes
34 -----
38 -----
35 The calculation for Cartesian coordinate is adapted from equations
39 The calculation for Cartesian coordinate is adapted from equations
36 2.28(b) and 2.28(c) of Doviak and Zrnic [1]_ assuming a
40 2.28(b) and 2.28(c) of Doviak and Zrnic [1]_ assuming a
37 standard atmosphere (4/3 Earth's radius model).
41 standard atmosphere (4/3 Earth's radius model).
38
42
39 .. math::
43 .. math::
40
44
41 z = \\sqrt{r^2+R^2+2*r*R*sin(\\theta_e)} - R
45 z = \\sqrt{r^2+R^2+2*r*R*sin(\\theta_e)} - R
42
46
43 s = R * arcsin(\\frac{r*cos(\\theta_e)}{R+z})
47 s = R * arcsin(\\frac{r*cos(\\theta_e)}{R+z})
44
48
45 x = s * sin(\\theta_a)
49 x = s * sin(\\theta_a)
46
50
47 y = s * cos(\\theta_a)
51 y = s * cos(\\theta_a)
48
52
49 Where r is the distance from the radar to the center of the gate,
53 Where r is the distance from the radar to the center of the gate,
50 :math:`\\theta_a` is the azimuth angle, :math:`\\theta_e` is the
54 :math:`\\theta_a` is the azimuth angle, :math:`\\theta_e` is the
51 elevation angle, s is the arc length, and R is the effective radius
55 elevation angle, s is the arc length, and R is the effective radius
52 of the earth, taken to be 4/3 the mean radius of earth (6371 km).
56 of the earth, taken to be 4/3 the mean radius of earth (6371 km).
53
57
54 References
58 References
55 ----------
59 ----------
56 .. [1] Doviak and Zrnic, Doppler Radar and Weather Observations, Second
60 .. [1] Doviak and Zrnic, Doppler Radar and Weather Observations, Second
57 Edition, 1993, p. 21.
61 Edition, 1993, p. 21.
58
62
59 """
63 """
60 theta_e = numpy.deg2rad(elevations) # elevation angle in radians.
64 theta_e = numpy.deg2rad(elevations) # elevation angle in radians.
61 theta_a = numpy.deg2rad(azimuths) # azimuth angle in radians.
65 theta_a = numpy.deg2rad(azimuths) # azimuth angle in radians.
62 R = 6371.0 * 1000.0 * 4.0 / 3.0 # effective radius of earth in meters.
66 R = 6371.0 * 1000.0 * 4.0 / 3.0 # effective radius of earth in meters.
63 r = ranges * 1000.0 # distances to gates in meters.
67 r = ranges * 1000.0 # distances to gates in meters.
64
68
65 z = (r ** 2 + R ** 2 + 2.0 * r * R * numpy.sin(theta_e)) ** 0.5 - R
69 z = (r ** 2 + R ** 2 + 2.0 * r * R * numpy.sin(theta_e)) ** 0.5 - R
66 s = R * numpy.arcsin(r * numpy.cos(theta_e) / (R + z)) # arc length in m.
70 s = R * numpy.arcsin(r * numpy.cos(theta_e) / (R + z)) # arc length in m.
67 x = s * numpy.sin(theta_a)
71 x = s * numpy.sin(theta_a)
68 y = s * numpy.cos(theta_a)
72 y = s * numpy.cos(theta_a)
69 return x, y, z
73 return x, y, z
70
74
71 def cartesian_to_geographic_aeqd(x, y, lon_0, lat_0, R=EARTH_RADIUS):
75 def cartesian_to_geographic_aeqd(x, y, lon_0, lat_0, R=EARTH_RADIUS):
72 """
76 """
73 Azimuthal equidistant Cartesian to geographic coordinate transform.
77 Azimuthal equidistant Cartesian to geographic coordinate transform.
74
78
75 Transform a set of Cartesian/Cartographic coordinates (x, y) to
79 Transform a set of Cartesian/Cartographic coordinates (x, y) to
76 geographic coordinate system (lat, lon) using a azimuthal equidistant
80 geographic coordinate system (lat, lon) using a azimuthal equidistant
77 map projection [1]_.
81 map projection [1]_.
78
82
79 .. math::
83 .. math::
80
84
81 lat = \\arcsin(\\cos(c) * \\sin(lat_0) +
85 lat = \\arcsin(\\cos(c) * \\sin(lat_0) +
82 (y * \\sin(c) * \\cos(lat_0) / \\rho))
86 (y * \\sin(c) * \\cos(lat_0) / \\rho))
83
87
84 lon = lon_0 + \\arctan2(
88 lon = lon_0 + \\arctan2(
85 x * \\sin(c),
89 x * \\sin(c),
86 \\rho * \\cos(lat_0) * \\cos(c) - y * \\sin(lat_0) * \\sin(c))
90 \\rho * \\cos(lat_0) * \\cos(c) - y * \\sin(lat_0) * \\sin(c))
87
91
88 \\rho = \\sqrt(x^2 + y^2)
92 \\rho = \\sqrt(x^2 + y^2)
89
93
90 c = \\rho / R
94 c = \\rho / R
91
95
92 Where x, y are the Cartesian position from the center of projection;
96 Where x, y are the Cartesian position from the center of projection;
93 lat, lon the corresponding latitude and longitude; lat_0, lon_0 are the
97 lat, lon the corresponding latitude and longitude; lat_0, lon_0 are the
94 latitude and longitude of the center of the projection; R is the radius of
98 latitude and longitude of the center of the projection; R is the radius of
95 the earth (defaults to ~6371 km). lon is adjusted to be between -180 and
99 the earth (defaults to ~6371 km). lon is adjusted to be between -180 and
96 180.
100 180.
97
101
98 Parameters
102 Parameters
99 ----------
103 ----------
100 x, y : array-like
104 x, y : array-like
101 Cartesian coordinates in the same units as R, typically meters.
105 Cartesian coordinates in the same units as R, typically meters.
102 lon_0, lat_0 : float
106 lon_0, lat_0 : float
103 Longitude and latitude, in degrees, of the center of the projection.
107 Longitude and latitude, in degrees, of the center of the projection.
104 R : float, optional
108 R : float, optional
105 Earth radius in the same units as x and y. The default value is in
109 Earth radius in the same units as x and y. The default value is in
106 units of meters.
110 units of meters.
107
111
108 Returns
112 Returns
109 -------
113 -------
110 lon, lat : array
114 lon, lat : array
111 Longitude and latitude of Cartesian coordinates in degrees.
115 Longitude and latitude of Cartesian coordinates in degrees.
112
116
113 References
117 References
114 ----------
118 ----------
115 .. [1] Snyder, J. P. Map Projections--A Working Manual. U. S. Geological
119 .. [1] Snyder, J. P. Map Projections--A Working Manual. U. S. Geological
116 Survey Professional Paper 1395, 1987, pp. 191-202.
120 Survey Professional Paper 1395, 1987, pp. 191-202.
117
121
118 """
122 """
119 x = numpy.atleast_1d(numpy.asarray(x))
123 x = numpy.atleast_1d(numpy.asarray(x))
120 y = numpy.atleast_1d(numpy.asarray(y))
124 y = numpy.atleast_1d(numpy.asarray(y))
121
125
122 lat_0_rad = numpy.deg2rad(lat_0)
126 lat_0_rad = numpy.deg2rad(lat_0)
123 lon_0_rad = numpy.deg2rad(lon_0)
127 lon_0_rad = numpy.deg2rad(lon_0)
124
128
125 rho = numpy.sqrt(x*x + y*y)
129 rho = numpy.sqrt(x*x + y*y)
126 c = rho / R
130 c = rho / R
127
131
128 with warnings.catch_warnings():
132 with warnings.catch_warnings():
129 # division by zero may occur here but is properly addressed below so
133 # division by zero may occur here but is properly addressed below so
130 # the warnings can be ignored
134 # the warnings can be ignored
131 warnings.simplefilter("ignore", RuntimeWarning)
135 warnings.simplefilter("ignore", RuntimeWarning)
132 lat_rad = numpy.arcsin(numpy.cos(c) * numpy.sin(lat_0_rad) +
136 lat_rad = numpy.arcsin(numpy.cos(c) * numpy.sin(lat_0_rad) +
133 y * numpy.sin(c) * numpy.cos(lat_0_rad) / rho)
137 y * numpy.sin(c) * numpy.cos(lat_0_rad) / rho)
134 lat_deg = numpy.rad2deg(lat_rad)
138 lat_deg = numpy.rad2deg(lat_rad)
135 # fix cases where the distance from the center of the projection is zero
139 # fix cases where the distance from the center of the projection is zero
136 lat_deg[rho == 0] = lat_0
140 lat_deg[rho == 0] = lat_0
137
141
138 x1 = x * numpy.sin(c)
142 x1 = x * numpy.sin(c)
139 x2 = rho*numpy.cos(lat_0_rad)*numpy.cos(c) - y*numpy.sin(lat_0_rad)*numpy.sin(c)
143 x2 = rho*numpy.cos(lat_0_rad)*numpy.cos(c) - y*numpy.sin(lat_0_rad)*numpy.sin(c)
140 lon_rad = lon_0_rad + numpy.arctan2(x1, x2)
144 lon_rad = lon_0_rad + numpy.arctan2(x1, x2)
141 lon_deg = numpy.rad2deg(lon_rad)
145 lon_deg = numpy.rad2deg(lon_rad)
142 # Longitudes should be from -180 to 180 degrees
146 # Longitudes should be from -180 to 180 degrees
143 lon_deg[lon_deg > 180] -= 360.
147 lon_deg[lon_deg > 180] -= 360.
144 lon_deg[lon_deg < -180] += 360.
148 lon_deg[lon_deg < -180] += 360.
145
149
146 return lon_deg, lat_deg
150 return lon_deg, lat_deg
147
151
148 def antenna_to_geographic(ranges, azimuths, elevations, site):
152 def antenna_to_geographic(ranges, azimuths, elevations, site):
149
153
150 x, y, z = antenna_to_cartesian(numpy.array(ranges), numpy.array(azimuths), numpy.array(elevations))
154 x, y, z = antenna_to_cartesian(numpy.array(ranges), numpy.array(azimuths), numpy.array(elevations))
151 lon, lat = cartesian_to_geographic_aeqd(x, y, site[0], site[1], R=6370997.)
155 lon, lat = cartesian_to_geographic_aeqd(x, y, site[0], site[1], R=6370997.)
152
156
153 return lon, lat
157 return lon, lat
154
158
155 def ll2xy(lat1, lon1, lat2, lon2):
159 def ll2xy(lat1, lon1, lat2, lon2):
156
160
157 p = 0.017453292519943295
161 p = 0.017453292519943295
158 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
162 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
159 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
163 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
160 r = 12742 * numpy.arcsin(numpy.sqrt(a))
164 r = 12742 * numpy.arcsin(numpy.sqrt(a))
161 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
165 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
162 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
166 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
163 theta = -theta + numpy.pi/2
167 theta = -theta + numpy.pi/2
164 return r*numpy.cos(theta), r*numpy.sin(theta)
168 return r*numpy.cos(theta), r*numpy.sin(theta)
165
169
166
170
167 def km2deg(km):
171 def km2deg(km):
168 '''
172 '''
169 Convert distance in km to degrees
173 Convert distance in km to degrees
170 '''
174 '''
171
175
172 return numpy.rad2deg(km/EARTH_RADIUS)
176 return numpy.rad2deg(km/EARTH_RADIUS)
173
177
174
178
175
179
176 class SpectralMomentsPlot(SpectraPlot):
180 class SpectralMomentsPlot(SpectraPlot):
177 '''
181 '''
178 Plot for Spectral Moments
182 Plot for Spectral Moments
179 '''
183 '''
180 CODE = 'spc_moments'
184 CODE = 'spc_moments'
181 # colormap = 'jet'
185 # colormap = 'jet'
182 # plot_type = 'pcolor'
186 # plot_type = 'pcolor'
183
187
184 class DobleGaussianPlot(SpectraPlot):
188 class DobleGaussianPlot(SpectraPlot):
185 '''
189 '''
186 Plot for Double Gaussian Plot
190 Plot for Double Gaussian Plot
187 '''
191 '''
188 CODE = 'gaussian_fit'
192 CODE = 'gaussian_fit'
189 # colormap = 'jet'
193 # colormap = 'jet'
190 # plot_type = 'pcolor'
194 # plot_type = 'pcolor'
191
195
192 class DoubleGaussianSpectraCutPlot(SpectraCutPlot):
196 class DoubleGaussianSpectraCutPlot(SpectraCutPlot):
193 '''
197 '''
194 Plot SpectraCut with Double Gaussian Fit
198 Plot SpectraCut with Double Gaussian Fit
195 '''
199 '''
196 CODE = 'cut_gaussian_fit'
200 CODE = 'cut_gaussian_fit'
197
201
198 class SnrPlot(RTIPlot):
202 class SnrPlot(RTIPlot):
199 '''
203 '''
200 Plot for SNR Data
204 Plot for SNR Data
201 '''
205 '''
202
206
203 CODE = 'snr'
207 CODE = 'snr'
204 colormap = 'jet'
208 colormap = 'jet'
205
209
206 def update(self, dataOut):
210 def update(self, dataOut):
207
211
208 data = {
212 data = {
209 'snr': 10*numpy.log10(dataOut.data_snr)
213 'snr': 10*numpy.log10(dataOut.data_snr)
210 }
214 }
211
215
212 return data, {}
216 return data, {}
213
217
214 class DopplerPlot(RTIPlot):
218 class DopplerPlot(RTIPlot):
215 '''
219 '''
216 Plot for DOPPLER Data (1st moment)
220 Plot for DOPPLER Data (1st moment)
217 '''
221 '''
218
222
219 CODE = 'dop'
223 CODE = 'dop'
220 colormap = 'jet'
224 colormap = 'jet'
221
225
222 def update(self, dataOut):
226 def update(self, dataOut):
223
227
224 data = {
228 data = {
225 'dop': 10*numpy.log10(dataOut.data_dop)
229 'dop': 10*numpy.log10(dataOut.data_dop)
226 }
230 }
227
231
228 return data, {}
232 return data, {}
229
233
230 class PowerPlot(RTIPlot):
234 class PowerPlot(RTIPlot):
231 '''
235 '''
232 Plot for Power Data (0 moment)
236 Plot for Power Data (0 moment)
233 '''
237 '''
234
238
235 CODE = 'pow'
239 CODE = 'pow'
236 colormap = 'jet'
240 colormap = 'jet'
237
241
238 def update(self, dataOut):
242 def update(self, dataOut):
239 data = {
243 data = {
240 'pow': 10*numpy.log10(dataOut.data_pow/dataOut.normFactor)
244 'pow': 10*numpy.log10(dataOut.data_pow/dataOut.normFactor)
241 }
245 }
242 return data, {}
246 return data, {}
243
247
244 class SpectralWidthPlot(RTIPlot):
248 class SpectralWidthPlot(RTIPlot):
245 '''
249 '''
246 Plot for Spectral Width Data (2nd moment)
250 Plot for Spectral Width Data (2nd moment)
247 '''
251 '''
248
252
249 CODE = 'width'
253 CODE = 'width'
250 colormap = 'jet'
254 colormap = 'jet'
251
255
252 def update(self, dataOut):
256 def update(self, dataOut):
253
257
254 data = {
258 data = {
255 'width': dataOut.data_width
259 'width': dataOut.data_width
256 }
260 }
257
261
258 return data, {}
262 return data, {}
259
263
260 class SkyMapPlot(Plot):
264 class SkyMapPlot(Plot):
261 '''
265 '''
262 Plot for meteors detection data
266 Plot for meteors detection data
263 '''
267 '''
264
268
265 CODE = 'param'
269 CODE = 'param'
266
270
267 def setup(self):
271 def setup(self):
268
272
269 self.ncols = 1
273 self.ncols = 1
270 self.nrows = 1
274 self.nrows = 1
271 self.width = 7.2
275 self.width = 7.2
272 self.height = 7.2
276 self.height = 7.2
273 self.nplots = 1
277 self.nplots = 1
274 self.xlabel = 'Zonal Zenith Angle (deg)'
278 self.xlabel = 'Zonal Zenith Angle (deg)'
275 self.ylabel = 'Meridional Zenith Angle (deg)'
279 self.ylabel = 'Meridional Zenith Angle (deg)'
276 self.polar = True
280 self.polar = True
277 self.ymin = -180
281 self.ymin = -180
278 self.ymax = 180
282 self.ymax = 180
279 self.colorbar = False
283 self.colorbar = False
280
284
281 def plot(self):
285 def plot(self):
282
286
283 arrayParameters = numpy.concatenate(self.data['param'])
287 arrayParameters = numpy.concatenate(self.data['param'])
284 error = arrayParameters[:, -1]
288 error = arrayParameters[:, -1]
285 indValid = numpy.where(error == 0)[0]
289 indValid = numpy.where(error == 0)[0]
286 finalMeteor = arrayParameters[indValid, :]
290 finalMeteor = arrayParameters[indValid, :]
287 finalAzimuth = finalMeteor[:, 3]
291 finalAzimuth = finalMeteor[:, 3]
288 finalZenith = finalMeteor[:, 4]
292 finalZenith = finalMeteor[:, 4]
289
293
290 x = finalAzimuth * numpy.pi / 180
294 x = finalAzimuth * numpy.pi / 180
291 y = finalZenith
295 y = finalZenith
292
296
293 ax = self.axes[0]
297 ax = self.axes[0]
294
298
295 if ax.firsttime:
299 if ax.firsttime:
296 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
300 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
297 else:
301 else:
298 ax.plot.set_data(x, y)
302 ax.plot.set_data(x, y)
299
303
300 dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
304 dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
301 dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
305 dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
302 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
306 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
303 dt2,
307 dt2,
304 len(x))
308 len(x))
305 self.titles[0] = title
309 self.titles[0] = title
306
310
307
311
308 class GenericRTIPlot(Plot):
312 class GenericRTIPlot(Plot):
309 '''
313 '''
310 Plot for data_xxxx object
314 Plot for data_xxxx object
311 '''
315 '''
312
316
313 CODE = 'param'
317 CODE = 'param'
314 colormap = 'viridis'
318 colormap = 'viridis'
315 plot_type = 'pcolorbuffer'
319 plot_type = 'pcolorbuffer'
316
320
317 def setup(self):
321 def setup(self):
318 self.xaxis = 'time'
322 self.xaxis = 'time'
319 self.ncols = 1
323 self.ncols = 1
320 self.nrows = self.data.shape('param')[0]
324 self.nrows = self.data.shape('param')[0]
321 self.nplots = self.nrows
325 self.nplots = self.nrows
322 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
326 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
323
327
324 if not self.xlabel:
328 if not self.xlabel:
325 self.xlabel = 'Time'
329 self.xlabel = 'Time'
326
330
327 self.ylabel = 'Range [km]'
331 self.ylabel = 'Range [km]'
328 if not self.titles:
332 if not self.titles:
329 self.titles = ['Param {}'.format(x) for x in range(self.nrows)]
333 self.titles = ['Param {}'.format(x) for x in range(self.nrows)]
330
334
331 def update(self, dataOut):
335 def update(self, dataOut):
332
336
333 data = {
337 data = {
334 'param' : numpy.concatenate([getattr(dataOut, attr) for attr in self.attr_data], axis=0)
338 'param' : numpy.concatenate([getattr(dataOut, attr) for attr in self.attr_data], axis=0)
335 }
339 }
336
340
337 meta = {}
341 meta = {}
338
342
339 return data, meta
343 return data, meta
340
344
341 def plot(self):
345 def plot(self):
342 # self.data.normalize_heights()
346 # self.data.normalize_heights()
343 self.x = self.data.times
347 self.x = self.data.times
344 self.y = self.data.yrange
348 self.y = self.data.yrange
345 self.z = self.data['param']
349 self.z = self.data['param']
346 self.z = 10*numpy.log10(self.z)
350 self.z = 10*numpy.log10(self.z)
347 self.z = numpy.ma.masked_invalid(self.z)
351 self.z = numpy.ma.masked_invalid(self.z)
348
352
349 if self.decimation is None:
353 if self.decimation is None:
350 x, y, z = self.fill_gaps(self.x, self.y, self.z)
354 x, y, z = self.fill_gaps(self.x, self.y, self.z)
351 else:
355 else:
352 x, y, z = self.fill_gaps(*self.decimate())
356 x, y, z = self.fill_gaps(*self.decimate())
353
357
354 for n, ax in enumerate(self.axes):
358 for n, ax in enumerate(self.axes):
355
359
356 self.zmax = self.zmax if self.zmax is not None else numpy.max(
360 self.zmax = self.zmax if self.zmax is not None else numpy.max(
357 self.z[n])
361 self.z[n])
358 self.zmin = self.zmin if self.zmin is not None else numpy.min(
362 self.zmin = self.zmin if self.zmin is not None else numpy.min(
359 self.z[n])
363 self.z[n])
360
364
361 if ax.firsttime:
365 if ax.firsttime:
362 if self.zlimits is not None:
366 if self.zlimits is not None:
363 self.zmin, self.zmax = self.zlimits[n]
367 self.zmin, self.zmax = self.zlimits[n]
364
368
365 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
369 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
366 vmin=self.zmin,
370 vmin=self.zmin,
367 vmax=self.zmax,
371 vmax=self.zmax,
368 cmap=self.cmaps[n]
372 cmap=self.cmaps[n]
369 )
373 )
370 else:
374 else:
371 if self.zlimits is not None:
375 if self.zlimits is not None:
372 self.zmin, self.zmax = self.zlimits[n]
376 self.zmin, self.zmax = self.zlimits[n]
373 ax.collections.remove(ax.collections[0])
377 ax.collections.remove(ax.collections[0])
374 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
378 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
375 vmin=self.zmin,
379 vmin=self.zmin,
376 vmax=self.zmax,
380 vmax=self.zmax,
377 cmap=self.cmaps[n]
381 cmap=self.cmaps[n]
378 )
382 )
379
383
380
384
381 class PolarMapPlot(Plot):
385 class PolarMapPlot(Plot):
382 '''
386 '''
383 Plot for weather radar
387 Plot for weather radar
384 '''
388 '''
385
389
386 CODE = 'param'
390 CODE = 'param'
387 colormap = 'seismic'
391 colormap = 'seismic'
388
392
389 def setup(self):
393 def setup(self):
390 self.ncols = 1
394 self.ncols = 1
391 self.nrows = 1
395 self.nrows = 1
392 self.width = 9
396 self.width = 9
393 self.height = 8
397 self.height = 8
394 self.mode = self.data.meta['mode']
398 self.mode = self.data.meta['mode']
395 if self.channels is not None:
399 if self.channels is not None:
396 self.nplots = len(self.channels)
400 self.nplots = len(self.channels)
397 self.nrows = len(self.channels)
401 self.nrows = len(self.channels)
398 else:
402 else:
399 self.nplots = self.data.shape(self.CODE)[0]
403 self.nplots = self.data.shape(self.CODE)[0]
400 self.nrows = self.nplots
404 self.nrows = self.nplots
401 self.channels = list(range(self.nplots))
405 self.channels = list(range(self.nplots))
402 if self.mode == 'E':
406 if self.mode == 'E':
403 self.xlabel = 'Longitude'
407 self.xlabel = 'Longitude'
404 self.ylabel = 'Latitude'
408 self.ylabel = 'Latitude'
405 else:
409 else:
406 self.xlabel = 'Range (km)'
410 self.xlabel = 'Range (km)'
407 self.ylabel = 'Height (km)'
411 self.ylabel = 'Height (km)'
408 self.bgcolor = 'white'
412 self.bgcolor = 'white'
409 self.cb_labels = self.data.meta['units']
413 self.cb_labels = self.data.meta['units']
410 self.lat = self.data.meta['latitude']
414 self.lat = self.data.meta['latitude']
411 self.lon = self.data.meta['longitude']
415 self.lon = self.data.meta['longitude']
412 self.xmin, self.xmax = float(
416 self.xmin, self.xmax = float(
413 km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
417 km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
414 self.ymin, self.ymax = float(
418 self.ymin, self.ymax = float(
415 km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
419 km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
416 # self.polar = True
420 # self.polar = True
417
421
418 def plot(self):
422 def plot(self):
419
423
420 for n, ax in enumerate(self.axes):
424 for n, ax in enumerate(self.axes):
421 data = self.data['param'][self.channels[n]]
425 data = self.data['param'][self.channels[n]]
422
426
423 zeniths = numpy.linspace(
427 zeniths = numpy.linspace(
424 0, self.data.meta['max_range'], data.shape[1])
428 0, self.data.meta['max_range'], data.shape[1])
425 if self.mode == 'E':
429 if self.mode == 'E':
426 azimuths = -numpy.radians(self.data.yrange)+numpy.pi/2
430 azimuths = -numpy.radians(self.data.yrange)+numpy.pi/2
427 r, theta = numpy.meshgrid(zeniths, azimuths)
431 r, theta = numpy.meshgrid(zeniths, azimuths)
428 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
432 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
429 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
433 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
430 x = km2deg(x) + self.lon
434 x = km2deg(x) + self.lon
431 y = km2deg(y) + self.lat
435 y = km2deg(y) + self.lat
432 else:
436 else:
433 azimuths = numpy.radians(self.data.yrange)
437 azimuths = numpy.radians(self.data.yrange)
434 r, theta = numpy.meshgrid(zeniths, azimuths)
438 r, theta = numpy.meshgrid(zeniths, azimuths)
435 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
439 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
436 self.y = zeniths
440 self.y = zeniths
437
441
438 if ax.firsttime:
442 if ax.firsttime:
439 if self.zlimits is not None:
443 if self.zlimits is not None:
440 self.zmin, self.zmax = self.zlimits[n]
444 self.zmin, self.zmax = self.zlimits[n]
441 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
445 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
442 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
446 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
443 vmin=self.zmin,
447 vmin=self.zmin,
444 vmax=self.zmax,
448 vmax=self.zmax,
445 cmap=self.cmaps[n])
449 cmap=self.cmaps[n])
446 else:
450 else:
447 if self.zlimits is not None:
451 if self.zlimits is not None:
448 self.zmin, self.zmax = self.zlimits[n]
452 self.zmin, self.zmax = self.zlimits[n]
449 ax.collections.remove(ax.collections[0])
453 ax.collections.remove(ax.collections[0])
450 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
454 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
451 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
455 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
452 vmin=self.zmin,
456 vmin=self.zmin,
453 vmax=self.zmax,
457 vmax=self.zmax,
454 cmap=self.cmaps[n])
458 cmap=self.cmaps[n])
455
459
456 if self.mode == 'A':
460 if self.mode == 'A':
457 continue
461 continue
458
462
459 # plot district names
463 # plot district names
460 f = open('/data/workspace/schain_scripts/distrito.csv')
464 f = open('/data/workspace/schain_scripts/distrito.csv')
461 for line in f:
465 for line in f:
462 label, lon, lat = [s.strip() for s in line.split(',') if s]
466 label, lon, lat = [s.strip() for s in line.split(',') if s]
463 lat = float(lat)
467 lat = float(lat)
464 lon = float(lon)
468 lon = float(lon)
465 # ax.plot(lon, lat, '.b', ms=2)
469 # ax.plot(lon, lat, '.b', ms=2)
466 ax.text(lon, lat, label.decode('utf8'), ha='center',
470 ax.text(lon, lat, label.decode('utf8'), ha='center',
467 va='bottom', size='8', color='black')
471 va='bottom', size='8', color='black')
468
472
469 # plot limites
473 # plot limites
470 limites = []
474 limites = []
471 tmp = []
475 tmp = []
472 for line in open('/data/workspace/schain_scripts/lima.csv'):
476 for line in open('/data/workspace/schain_scripts/lima.csv'):
473 if '#' in line:
477 if '#' in line:
474 if tmp:
478 if tmp:
475 limites.append(tmp)
479 limites.append(tmp)
476 tmp = []
480 tmp = []
477 continue
481 continue
478 values = line.strip().split(',')
482 values = line.strip().split(',')
479 tmp.append((float(values[0]), float(values[1])))
483 tmp.append((float(values[0]), float(values[1])))
480 for points in limites:
484 for points in limites:
481 ax.add_patch(
485 ax.add_patch(
482 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
486 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
483
487
484 # plot Cuencas
488 # plot Cuencas
485 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
489 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
486 f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
490 f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
487 values = [line.strip().split(',') for line in f]
491 values = [line.strip().split(',') for line in f]
488 points = [(float(s[0]), float(s[1])) for s in values]
492 points = [(float(s[0]), float(s[1])) for s in values]
489 ax.add_patch(Polygon(points, ec='b', fc='none'))
493 ax.add_patch(Polygon(points, ec='b', fc='none'))
490
494
491 # plot grid
495 # plot grid
492 for r in (15, 30, 45, 60):
496 for r in (15, 30, 45, 60):
493 ax.add_artist(plt.Circle((self.lon, self.lat),
497 ax.add_artist(plt.Circle((self.lon, self.lat),
494 km2deg(r), color='0.6', fill=False, lw=0.2))
498 km2deg(r), color='0.6', fill=False, lw=0.2))
495 ax.text(
499 ax.text(
496 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
500 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
497 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
501 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
498 '{}km'.format(r),
502 '{}km'.format(r),
499 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
503 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
500
504
501 if self.mode == 'E':
505 if self.mode == 'E':
502 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
506 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
503 label = 'E{:02d}'.format(int(self.data.meta['elevation']))
507 label = 'E{:02d}'.format(int(self.data.meta['elevation']))
504 else:
508 else:
505 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
509 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
506 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
510 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
507
511
508 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
512 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
509 self.titles = ['{} {}'.format(
513 self.titles = ['{} {}'.format(
510 self.data.parameters[x], title) for x in self.channels]
514 self.data.parameters[x], title) for x in self.channels]
511
515
512 class WeatherParamsPlot(Plot):
516 class WeatherParamsPlot(Plot):
513 #CODE = 'RHI'
517 #CODE = 'RHI'
514 #plot_name = 'RHI'
518 #plot_name = 'RHI'
515 plot_type = 'scattermap'
519 plot_type = 'scattermap'
516 buffering = False
520 buffering = False
521 projection = ccrs.PlateCarree()
517
522
518 def setup(self):
523 def setup(self):
519
524
520 self.ncols = 1
525 self.ncols = 1
521 self.nrows = 1
526 self.nrows = 1
522 self.nplots= 1
527 self.nplots= 1
523 self.ylabel= 'Range [km]'
528 self.ylabel= 'Range [km]'
524 self.xlabel= 'Range [km]'
529 self.xlabel= 'Range [km]'
525 self.polar = True
530
526 self.grid = True
527 if self.channels is not None:
531 if self.channels is not None:
528 self.nplots = len(self.channels)
532 self.nplots = len(self.channels)
529 self.ncols = len(self.channels)
533 self.ncols = len(self.channels)
530 else:
534 else:
531 self.nplots = self.data.shape(self.CODE)[0]
535 self.nplots = self.data.shape(self.CODE)[0]
532 self.ncols = self.nplots
536 self.ncols = self.nplots
533 self.channels = list(range(self.nplots))
537 self.channels = list(range(self.nplots))
534
538
535 self.colorbar=True
539 self.colorbar=True
536 if len(self.channels)>1:
540 if len(self.channels)>1:
537 self.width = 12
541 self.width = 12
538 else:
542 else:
539 self.width =8
543 self.width =8
540 self.height =8
544 self.height =7
541 self.ini =0
545 self.ini =0
542 self.len_azi =0
546 self.len_azi =0
543 self.buffer_ini = None
547 self.buffer_ini = None
544 self.buffer_ele = None
548 self.buffer_ele = None
545 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08})
549 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08})
546 self.flag =0
550 self.flag =0
547 self.indicador= 0
551 self.indicador= 0
548 self.last_data_ele = None
552 self.last_data_ele = None
549 self.val_mean = None
553 self.val_mean = None
550
554
551 def update(self, dataOut):
555 def update(self, dataOut):
552
556
553 vars = {
557 vars = {
554 'S' : 0,
558 'S' : 0,
555 'V' : 1,
559 'V' : 1,
556 'W' : 2,
560 'W' : 2,
557 'SNR' : 3,
561 'SNR' : 3,
558 'Z' : 4,
562 'Z' : 4,
559 'D' : 5,
563 'D' : 5,
560 'P' : 6,
564 'P' : 6,
561 'R' : 7,
565 'R' : 7,
562 }
566 }
563
567
564 data = {}
568 data = {}
565 meta = {}
569 meta = {}
566
570
567 if hasattr(dataOut, 'nFFTPoints'):
571 if hasattr(dataOut, 'nFFTPoints'):
568 factor = dataOut.normFactor
572 factor = dataOut.normFactor
569 else:
573 else:
570 factor = 1
574 factor = 1
571
575
572 if hasattr(dataOut, 'dparam'):
576 if hasattr(dataOut, 'dparam'):
573 tmp = getattr(dataOut, 'data_param')
577 tmp = getattr(dataOut, 'data_param')
574 else:
578 else:
575
579
576 if 'S' in self.attr_data[0]:
580 if 'S' in self.attr_data[0]:
577 tmp = 10*numpy.log10(10.0*getattr(dataOut, 'data_param')[:,0,:]/(factor))
581 tmp = 10*numpy.log10(10.0*getattr(dataOut, 'data_param')[:,0,:]/(factor))
578 else:
582 else:
579 tmp = getattr(dataOut, 'data_param')[:,vars[self.attr_data[0]],:]
583 tmp = getattr(dataOut, 'data_param')[:,vars[self.attr_data[0]],:]
580
584
581 if self.mask:
585 if self.mask:
582 mask = dataOut.data_param[:,3,:] < self.mask
586 mask = dataOut.data_param[:,3,:] < self.mask
583 tmp = numpy.ma.masked_array(tmp, mask=mask)
587 tmp = numpy.ma.masked_array(tmp, mask=mask)
584
588
585 r = dataOut.heightList
589 r = dataOut.heightList
586 delta_height = r[1]-r[0]
590 delta_height = r[1]-r[0]
587 valid = numpy.where(r>=0)[0]
591 valid = numpy.where(r>=0)[0]
588 data['r'] = numpy.arange(len(valid))*delta_height
592 data['r'] = numpy.arange(len(valid))*delta_height
589
593
590 data['data'] = [0, 0]
594 data['data'] = [0, 0]
591
595
592 #try:
596 try:
593 data['data'][0] = tmp[0][:,valid]
597 data['data'][0] = tmp[0][:,valid]
594 data['data'][1] = tmp[1][:,valid]
598 data['data'][1] = tmp[1][:,valid]
595 #except:
599 except:
596 # data['data'] = tmp[0][:,valid]
600 data['data'][0] = tmp[0][:,valid]
601 data['data'][1] = tmp[0][:,valid]
597
602
598 if dataOut.mode_op == 'PPI':
603 if dataOut.mode_op == 'PPI':
599 self.CODE = 'PPI'
604 self.CODE = 'PPI'
600 self.title = self.CODE
605 self.title = self.CODE
601 elif dataOut.mode_op == 'RHI':
606 elif dataOut.mode_op == 'RHI':
602 self.CODE = 'RHI'
607 self.CODE = 'RHI'
603 self.title = self.CODE
608 self.title = self.CODE
604
609
605 data['azi'] = dataOut.data_azi
610 data['azi'] = dataOut.data_azi
606 data['ele'] = dataOut.data_ele
611 data['ele'] = dataOut.data_ele
607 data['mode_op'] = dataOut.mode_op
612 data['mode_op'] = dataOut.mode_op
608 self.mode = dataOut.mode_op
613 self.mode = dataOut.mode_op
609 var = data['data'][0].flatten()
610 r = numpy.tile(data['r'], data['data'][0].shape[0])
611 az = numpy.repeat(data['azi'], data['data'][0].shape[1])
612 el = numpy.repeat(data['ele'], data['data'][0].shape[1])
613
614 # lla = georef.spherical_to_proj(r, data['azi'], data['ele'], (-75.295893, -12.040436, 3379.2147))
615
616 latlon = antenna_to_geographic(r, az, el, (-75.295893, -12.040436))
617
618 if self.mask:
619 meta['lat'] = latlon[1][var.mask==False]
620 meta['lon'] = latlon[0][var.mask==False]
621 data['var'] = numpy.array([var[var.mask==False]])
622 else:
623 meta['lat'] = latlon[1]
624 meta['lon'] = latlon[0]
625 data['var'] = numpy.array([var])
626
614
627 return data, meta
615 return data, meta
628
616
629 def plot(self):
617 def plot(self):
630 data = self.data[-1]
618 data = self.data[-1]
631 z = data['data']
619 z = data['data']
632 r = data['r']
620 r = data['r']
633 self.titles = []
621 self.titles = []
634
622
635 self.ymax = self.ymax if self.ymax else numpy.nanmax(r)
623 self.ymax = self.ymax if self.ymax else numpy.nanmax(r)
636 self.ymin = self.ymin if self.ymin else numpy.nanmin(r)
624 self.ymin = self.ymin if self.ymin else numpy.nanmin(r)
637 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
625 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
638 self.zmin = self.zmin if self.zmin is not None else numpy.nanmin(z)
626 self.zmin = self.zmin if self.zmin is not None else numpy.nanmin(z)
639
627
640 if isinstance(data['mode_op'], bytes):
628 if isinstance(data['mode_op'], bytes):
641 data['mode_op'] = data['mode_op'].decode()
629 data['mode_op'] = data['mode_op'].decode()
642
630
643 if data['mode_op'] == 'RHI':
631 if data['mode_op'] == 'RHI':
644 try:
632 r, theta = numpy.meshgrid(r, numpy.radians(data['ele']))
645 if self.data['mode_op'][-2] == 'PPI':
633 len_aux = int(data['azi'].shape[0]/4)
646 self.ang_min = None
634 mean = numpy.mean(data['azi'][len_aux:-len_aux])
647 self.ang_max = None
635 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
648 except:
636 elif data['mode_op'] == 'PPI':
649 pass
637 r, theta = numpy.meshgrid(r, -numpy.radians(data['azi'])+numpy.pi/2)
650 self.ang_min = self.ang_min if self.ang_min else 0
638 len_aux = int(data['ele'].shape[0]/4)
651 self.ang_max = self.ang_max if self.ang_max else 90
639 mean = numpy.mean(data['ele'][len_aux:-len_aux])
652 r, theta = numpy.meshgrid(r, numpy.radians(data['ele']) )
640 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(mean)), r*numpy.sin(
653 elif data['mode_op'] == 'PPI':
641 theta)*numpy.cos(numpy.radians(mean))
654 try:
642 x = km2deg(x) + -75.295893
655 if self.data['mode_op'][-2] == 'RHI':
643 y = km2deg(y) + -12.040436
656 self.ang_min = None
657 self.ang_max = None
658 except:
659 pass
660 self.ang_min = self.ang_min if self.ang_min else 0
661 self.ang_max = self.ang_max if self.ang_max else 360
662 r, theta = numpy.meshgrid(r, numpy.radians(data['azi']) )
663
644
664 self.clear_figures()
645 self.clear_figures()
665
646
666 for i,ax in enumerate(self.axes):
647 if data['mode_op'] == 'PPI':
667
648 axes = self.axes['PPI']
668 if ax.firsttime:
649 else:
669 ax.set_xlim(numpy.radians(self.ang_min),numpy.radians(self.ang_max))
650 axes = self.axes['RHI']
670 ax.plt = ax.pcolormesh(theta, r, z[i], cmap=self.colormap, vmin=self.zmin, vmax=self.zmax)
671 if data['mode_op'] == 'PPI':
672 ax.set_theta_direction(-1)
673 ax.set_theta_offset(numpy.pi/2)
674
651
675 else:
652 for i, ax in enumerate(axes):
676 ax.set_xlim(numpy.radians(self.ang_min),numpy.radians(self.ang_max))
653 if data['mode_op'] == 'PPI':
677 ax.plt = ax.pcolormesh(theta, r, z[i], cmap=self.colormap, vmin=self.zmin, vmax=self.zmax)
654 ax.set_extent([-75.745893, -74.845893, -12.490436, -11.590436])
678 if data['mode_op'] == 'PPI':
655
679 ax.set_theta_direction(-1)
656 ax.plt = ax.pcolormesh(x, y, z[i], cmap=self.colormap, vmin=self.zmin, vmax=self.zmax)
680 ax.set_theta_offset(numpy.pi/2)
681
657
682 ax.grid(True)
683 if data['mode_op'] == 'RHI':
658 if data['mode_op'] == 'RHI':
684 len_aux = int(data['azi'].shape[0]/4)
659 len_aux = int(data['azi'].shape[0]/4)
685 mean = numpy.mean(data['azi'][len_aux:-len_aux])
660 mean = numpy.mean(data['azi'][len_aux:-len_aux])
686 if len(self.channels) !=1:
661 if len(self.channels) !=1:
687 self.titles = ['RHI {} at AZ: {} CH {}'.format(self.labels[x], str(round(mean,1)), x) for x in self.channels]
662 self.titles = ['RHI {} at AZ: {} CH {}'.format(self.labels[x], str(round(mean,1)), x) for x in self.channels]
688 else:
663 else:
689 self.titles = ['RHI {} at AZ: {} CH {}'.format(self.labels[0], str(round(mean,1)), self.channels[0])]
664 self.titles = ['RHI {} at AZ: {} CH {}'.format(self.labels[0], str(round(mean,1)), self.channels[0])]
690 elif data['mode_op'] == 'PPI':
665 elif data['mode_op'] == 'PPI':
691 len_aux = int(data['ele'].shape[0]/4)
666 len_aux = int(data['ele'].shape[0]/4)
692 mean = numpy.mean(data['ele'][len_aux:-len_aux])
667 mean = numpy.mean(data['ele'][len_aux:-len_aux])
693 if len(self.channels) !=1:
668 if len(self.channels) !=1:
694 self.titles = ['PPI {} at EL: {} CH {}'.format(self.labels[x], str(round(mean,1)), x) for x in self.channels]
669 self.titles = ['PPI {} at EL: {} CH {}'.format(self.labels[x], str(round(mean,1)), x) for x in self.channels]
695 else:
670 else:
696 self.titles = ['PPI {} at EL: {} CH {}'.format(self.labels[0], str(round(mean,1)), self.channels[0])]
671 self.titles = ['PPI {} at EL: {} CH {}'.format(self.labels[0], str(round(mean,1)), self.channels[0])]
697 self.mode_value = round(mean,1) No newline at end of file
672 self.mode_value = round(mean,1)
673
674 if data['mode_op'] == 'PPI':
675 gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True,
676 linewidth=1, color='gray', alpha=0.5, linestyle='--')
677 gl.xlabel_style = {'size': 8}
678 gl.ylabel_style = {'size': 8}
679 gl.xlabels_top = False
680 gl.ylabels_right = False
681 shape_p = os.path.join(self.shapes,'PER_ADM2/PER_ADM2.shp')
682 shape_d = os.path.join(self.shapes,'PER_ADM1/PER_ADM1.shp')
683 capitales = os.path.join(self.shapes,'CAPITALES/cap_provincia.shp')
684 vias = os.path.join(self.shapes,'Carreteras/VIAS_NACIONAL_250000.shp')
685 reader_d = shpreader.BasicReader(shape_p, encoding='latin1')
686 reader_p = shpreader.BasicReader(shape_d, encoding='latin1')
687 reader_c = shpreader.BasicReader(capitales, encoding='latin1')
688 reader_v = shpreader.BasicReader(vias, encoding='latin1')
689 caps = [x for x in reader_c.records() if x.attributes["Departa"] in ("JUNIN", "LIMA", "AYACUCHO", "HUANCAVELICA")]
690 districts = [x for x in reader_d.records() if x.attributes["Name"] in ("JUNÍN", "CHANCHAMAYO", "CHUPACA", "CONCEPCIÓN", "HUANCAYO", "JAUJA", "SATIPO", "TARMA", "YAUYOS", "HUAROCHIRÍ", "CANTA", "HUANTA", "TAYACAJA")]
691 provs = [x for x in reader_p.records() if x.attributes["NAME"] in ("Junín", "Lima")]
692 vias = [x for x in reader_v.records() if x.attributes["DEP"] in ("JUNIN", "LIMA")]
693
694 # Display Kenya's shape
695 shape_feature = ShapelyFeature([x.geometry for x in districts], ccrs.PlateCarree(), facecolor="none", edgecolor='grey', lw=0.5)
696 ax.add_feature(shape_feature)
697 shape_feature = ShapelyFeature([x.geometry for x in provs], ccrs.PlateCarree(), facecolor="none", edgecolor='white', lw=1)
698 ax.add_feature(shape_feature)
699 shape_feature = ShapelyFeature([x.geometry for x in vias], ccrs.PlateCarree(), facecolor="none", edgecolor='yellow', lw=1)
700 ax.add_feature(shape_feature)
701
702 for cap in caps:
703 if cap.attributes['Nombre'] in ("LA OROYA", "CONCEPCIÓN", "HUANCAYO", "JAUJA", "CHUPACA", "YAUYOS", "HUANTA", "PAMPAS"):
704 ax.text(cap.attributes['X'], cap.attributes['Y'], cap.attributes['Nombre'].title(), size=7, color='white')
705 ax.text(-75.052003, -11.915552, 'Huaytapallana', size=7, color='cyan')
706 ax.plot(-75.052003, -11.915552, '*')
707
708 for R in (10, 20, 30 , 40, 50):
709 circle = Circle((-75.295893, -12.040436), km2deg(R), facecolor='none',
710 edgecolor='skyblue', linewidth=1, alpha=0.5)
711 ax.add_patch(circle)
712 ax.text(km2deg(R)*numpy.cos(numpy.radians(45))-75.295893,
713 km2deg(R)*numpy.sin(numpy.radians(45))-12.040436,
714 '{}km'.format(R), color='skyblue', size=7)
715 elif data['mode_op'] == 'RHI':
716 ax.grid(color='grey', alpha=0.5, linestyle='--', linewidth=1)
Show file before | Show file after | Hide comments
@@ -1,733 +1,739
1 from email.utils import localtime
1 from email.utils import localtime
2 import os
2 import os
3 import time
3 import time
4 import datetime
4 import datetime
5
5
6 import numpy
6 import numpy
7 import h5py
7 import h5py
8
8
9 import schainpy.admin
9 import schainpy.admin
10 from schainpy.model.data.jrodata import *
10 from schainpy.model.data.jrodata import *
11 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
12 from schainpy.model.io.jroIO_base import *
12 from schainpy.model.io.jroIO_base import *
13 from schainpy.utils import log
13 from schainpy.utils import log
14
14
15
15
16 class HDFReader(Reader, ProcessingUnit):
16 class HDFReader(Reader, ProcessingUnit):
17 """Processing unit to read HDF5 format files
17 """Processing unit to read HDF5 format files
18
18
19 This unit reads HDF5 files created with `HDFWriter` operation contains
19 This unit reads HDF5 files created with `HDFWriter` operation contains
20 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 by default two groups Data and Metadata all variables would be saved as `dataOut`
21 attributes.
21 attributes.
22 It is possible to read any HDF5 file by given the structure in the `description`
22 It is possible to read any HDF5 file by given the structure in the `description`
23 parameter, also you can add extra values to metadata with the parameter `extras`.
23 parameter, also you can add extra values to metadata with the parameter `extras`.
24
24
25 Parameters:
25 Parameters:
26 -----------
26 -----------
27 path : str
27 path : str
28 Path where files are located.
28 Path where files are located.
29 startDate : date
29 startDate : date
30 Start date of the files
30 Start date of the files
31 endDate : list
31 endDate : list
32 End date of the files
32 End date of the files
33 startTime : time
33 startTime : time
34 Start time of the files
34 Start time of the files
35 endTime : time
35 endTime : time
36 End time of the files
36 End time of the files
37 description : dict, optional
37 description : dict, optional
38 Dictionary with the description of the HDF5 file
38 Dictionary with the description of the HDF5 file
39 extras : dict, optional
39 extras : dict, optional
40 Dictionary with extra metadata to be be added to `dataOut`
40 Dictionary with extra metadata to be be added to `dataOut`
41
41
42 Examples
42 Examples
43 --------
43 --------
44
44
45 desc = {
45 desc = {
46 'Data': {
46 'Data': {
47 'data_output': ['u', 'v', 'w'],
47 'data_output': ['u', 'v', 'w'],
48 'utctime': 'timestamps',
48 'utctime': 'timestamps',
49 } ,
49 } ,
50 'Metadata': {
50 'Metadata': {
51 'heightList': 'heights'
51 'heightList': 'heights'
52 }
52 }
53 }
53 }
54
54
55 desc = {
55 desc = {
56 'Data': {
56 'Data': {
57 'data_output': 'winds',
57 'data_output': 'winds',
58 'utctime': 'timestamps'
58 'utctime': 'timestamps'
59 },
59 },
60 'Metadata': {
60 'Metadata': {
61 'heightList': 'heights'
61 'heightList': 'heights'
62 }
62 }
63 }
63 }
64
64
65 extras = {
65 extras = {
66 'timeZone': 300
66 'timeZone': 300
67 }
67 }
68
68
69 reader = project.addReadUnit(
69 reader = project.addReadUnit(
70 name='HDFReader',
70 name='HDFReader',
71 path='/path/to/files',
71 path='/path/to/files',
72 startDate='2019/01/01',
72 startDate='2019/01/01',
73 endDate='2019/01/31',
73 endDate='2019/01/31',
74 startTime='00:00:00',
74 startTime='00:00:00',
75 endTime='23:59:59',
75 endTime='23:59:59',
76 # description=json.dumps(desc),
76 # description=json.dumps(desc),
77 # extras=json.dumps(extras),
77 # extras=json.dumps(extras),
78 )
78 )
79
79
80 """
80 """
81
81
82 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
83
83
84 def __init__(self):
84 def __init__(self):
85 ProcessingUnit.__init__(self)
85 ProcessingUnit.__init__(self)
86 self.dataOut = Parameters()
86 self.dataOut = Parameters()
87 self.ext = ".hdf5"
87 self.ext = ".hdf5"
88 self.optchar = "D"
88 self.optchar = "D"
89 self.meta = {}
89 self.meta = {}
90 self.data = {}
90 self.data = {}
91 self.open_file = h5py.File
91 self.open_file = h5py.File
92 self.open_mode = 'r'
92 self.open_mode = 'r'
93 self.description = {}
93 self.description = {}
94 self.extras = {}
94 self.extras = {}
95 self.filefmt = "*%Y%j***"
95 self.filefmt = "*%Y%j***"
96 self.folderfmt = "*%Y%j"
96 self.folderfmt = "*%Y%j"
97 self.utcoffset = 0
97 self.utcoffset = 0
98 self.filter = None
98 self.filter = None
99 self.dparam = None
99 self.dparam = None
100
100
101 def setup(self, **kwargs):
101 def setup(self, **kwargs):
102
102
103 self.set_kwargs(**kwargs)
103 self.set_kwargs(**kwargs)
104 if not self.ext.startswith('.'):
104 if not self.ext.startswith('.'):
105 self.ext = '.{}'.format(self.ext)
105 self.ext = '.{}'.format(self.ext)
106
106
107 if self.online:
107 if self.online:
108 log.log("Searching files in online mode...", self.name)
108 log.log("Searching files in online mode...", self.name)
109
109
110 for nTries in range(self.nTries):
110 for nTries in range(self.nTries):
111 fullpath = self.searchFilesOnLine(self.path, self.startDate,
111 fullpath = self.searchFilesOnLine(self.path, self.startDate,
112 self.endDate, self.expLabel, self.ext, self.walk,
112 self.endDate, self.expLabel, self.ext, self.walk,
113 self.filefmt, self.folderfmt,self.filter)
113 self.filefmt, self.folderfmt,self.filter)
114 try:
114 try:
115 fullpath = next(fullpath)
115 fullpath = next(fullpath)
116 except:
116 except:
117 fullpath = None
117 fullpath = None
118
118
119 if fullpath:
119 if fullpath:
120 break
120 break
121
121
122 log.warning(
122 log.warning(
123 'Waiting {} sec for a valid file in {}: try {} ...'.format(
123 'Waiting {} sec for a valid file in {}: try {} ...'.format(
124 self.delay, self.path, nTries + 1),
124 self.delay, self.path, nTries + 1),
125 self.name)
125 self.name)
126 time.sleep(self.delay)
126 time.sleep(self.delay)
127
127
128 if not(fullpath):
128 if not(fullpath):
129 raise schainpy.admin.SchainError(
129 raise schainpy.admin.SchainError(
130 'There isn\'t any valid file in {}'.format(self.path))
130 'There isn\'t any valid file in {}'.format(self.path))
131
131
132 pathname, filename = os.path.split(fullpath)
132 pathname, filename = os.path.split(fullpath)
133 self.year = int(filename[1:5])
133 self.year = int(filename[1:5])
134 self.doy = int(filename[5:8])
134 self.doy = int(filename[5:8])
135 self.set = int(filename[8:11]) - 1
135 self.set = int(filename[8:11]) - 1
136 else:
136 else:
137 log.log("Searching files in {}".format(self.path), self.name)
137 log.log("Searching files in {}".format(self.path), self.name)
138 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
138 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
139 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt,self.filter)
139 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt,self.filter)
140
140
141 self.setNextFile()
141 self.setNextFile()
142
142
143 return
143 return
144
144
145 def readFirstHeader(self):
145 def readFirstHeader(self):
146 '''Read metadata and data'''
146 '''Read metadata and data'''
147
147
148 self.__readMetadata()
148 self.__readMetadata()
149 self.__readData()
149 self.__readData()
150 self.__setBlockList()
150 self.__setBlockList()
151
151
152 if 'type' in self.meta:
152 if 'type' in self.meta:
153 self.dataOut = eval(self.meta['type'])()
153 self.dataOut = eval(self.meta['type'])()
154
154
155 if self.dparam:
155 if self.dparam:
156 setattr(self.dataOut, "dparam", 1)
156 setattr(self.dataOut, "dparam", 1)
157
157
158 for attr in self.meta:
158 for attr in self.meta:
159 setattr(self.dataOut, attr, self.meta[attr])
159 setattr(self.dataOut, attr, self.meta[attr])
160
160
161 self.blockIndex = 0
161 self.blockIndex = 0
162
162
163 return
163 return
164
164
165 def __setBlockList(self):
165 def __setBlockList(self):
166 '''
166 '''
167 Selects the data within the times defined
167 Selects the data within the times defined
168
168
169 self.fp
169 self.fp
170 self.startTime
170 self.startTime
171 self.endTime
171 self.endTime
172 self.blockList
172 self.blockList
173 self.blocksPerFile
173 self.blocksPerFile
174
174
175 '''
175 '''
176
176
177 startTime = self.startTime
177 startTime = self.startTime
178 endTime = self.endTime
178 endTime = self.endTime
179 thisUtcTime = self.data['utctime'] + self.utcoffset
179 thisUtcTime = self.data['utctime'] + self.utcoffset
180 try:
180 try:
181 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
181 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
182 except:
182 except:
183 self.interval = 0
183 self.interval = 0
184 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
184 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
185
185
186 thisDate = thisDatetime.date()
186 thisDate = thisDatetime.date()
187 thisTime = thisDatetime.time()
187 thisTime = thisDatetime.time()
188
188
189 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
189 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
190 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
190 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
191
191
192 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
192 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
193
193
194 self.blockList = ind
194 self.blockList = ind
195 self.blocksPerFile = len(ind)
195 self.blocksPerFile = len(ind)
196 return
196 return
197
197
198 def __readMetadata(self):
198 def __readMetadata(self):
199 '''
199 '''
200 Reads Metadata
200 Reads Metadata
201 '''
201 '''
202
202
203 meta = {}
203 meta = {}
204
204
205 if self.description:
205 if self.description:
206 for key, value in self.description['Metadata'].items():
206 for key, value in self.description['Metadata'].items():
207 meta[key] = self.fp[value][()]
207 meta[key] = self.fp[value][()]
208 else:
208 else:
209 grp = self.fp['Metadata']
209 grp = self.fp['Metadata']
210 for name in grp:
210 for name in grp:
211 meta[name] = grp[name][()]
211 meta[name] = grp[name][()]
212
212
213 if self.extras:
213 if self.extras:
214 for key, value in self.extras.items():
214 for key, value in self.extras.items():
215 meta[key] = value
215 meta[key] = value
216 self.meta = meta
216 self.meta = meta
217
217
218 return
218 return
219
219
220 def __readData(self):
220 def __readData(self):
221
221
222 data = {}
222 data = {}
223
223
224 if self.description:
224 if self.description:
225 for key, value in self.description['Data'].items():
225 for key, value in self.description['Data'].items():
226 if isinstance(value, str):
226 if isinstance(value, str):
227 if isinstance(self.fp[value], h5py.Dataset):
227 if isinstance(self.fp[value], h5py.Dataset):
228 data[key] = self.fp[value][()]
228 data[key] = self.fp[value][()]
229 elif isinstance(self.fp[value], h5py.Group):
229 elif isinstance(self.fp[value], h5py.Group):
230 array = []
230 array = []
231 for ch in self.fp[value]:
231 for ch in self.fp[value]:
232 array.append(self.fp[value][ch][()])
232 array.append(self.fp[value][ch][()])
233 data[key] = numpy.array(array)
233 data[key] = numpy.array(array)
234 elif isinstance(value, list):
234 elif isinstance(value, list):
235 array = []
235 array = []
236 for ch in value:
236 for ch in value:
237 array.append(self.fp[ch][()])
237 array.append(self.fp[ch][()])
238 data[key] = numpy.array(array)
238 data[key] = numpy.array(array)
239 else:
239 else:
240 grp = self.fp['Data']
240 grp = self.fp['Data']
241 for name in grp:
241 for name in grp:
242 if isinstance(grp[name], h5py.Dataset):
242 if isinstance(grp[name], h5py.Dataset):
243 array = grp[name][()]
243 array = grp[name][()]
244 elif isinstance(grp[name], h5py.Group):
244 elif isinstance(grp[name], h5py.Group):
245 array = []
245 array = []
246 for ch in grp[name]:
246 for ch in grp[name]:
247 array.append(grp[name][ch][()])
247 array.append(grp[name][ch][()])
248 array = numpy.array(array)
248 array = numpy.array(array)
249 else:
249 else:
250 log.warning('Unknown type: {}'.format(name))
250 log.warning('Unknown type: {}'.format(name))
251
251
252 if name in self.description:
252 if name in self.description:
253 key = self.description[name]
253 key = self.description[name]
254 else:
254 else:
255 key = name
255 key = name
256 data[key] = array
256 data[key] = array
257
257
258 self.data = data
258 self.data = data
259 return
259 return
260
260
261 def getData(self):
261 def getData(self):
262
262
263 for attr in self.data:
263 for attr in self.data:
264 if self.data[attr].ndim == 1:
264 if self.data[attr].ndim == 1:
265 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
265 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
266 else:
266 else:
267 if self.dparam:
267 if self.dparam:
268 setattr(self.dataOut, attr, self.data[attr])
268 setattr(self.dataOut, attr, self.data[attr])
269 else:
269 else:
270 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
270 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
271
271
272 self.dataOut.flagNoData = False
272 self.dataOut.flagNoData = False
273 self.blockIndex += 1
273 self.blockIndex += 1
274
274
275 log.log("Block No. {}/{} -> {}".format(
275 log.log("Block No. {}/{} -> {}".format(
276 self.blockIndex,
276 self.blockIndex,
277 self.blocksPerFile,
277 self.blocksPerFile,
278 self.dataOut.datatime.ctime()), self.name)
278 self.dataOut.datatime.ctime()), self.name)
279
279
280 return
280 return
281
281
282 def run(self, **kwargs):
282 def run(self, **kwargs):
283
283
284 if not(self.isConfig):
284 if not(self.isConfig):
285 self.setup(**kwargs)
285 self.setup(**kwargs)
286 self.isConfig = True
286 self.isConfig = True
287
287
288 if self.blockIndex == self.blocksPerFile:
288 if self.blockIndex == self.blocksPerFile:
289 self.setNextFile()
289 self.setNextFile()
290
290
291 self.getData()
291 self.getData()
292
292
293 return
293 return
294
294
295 @MPDecorator
295 @MPDecorator
296 class HDFWriter(Operation):
296 class HDFWriter(Operation):
297 """Operation to write HDF5 files.
297 """Operation to write HDF5 files.
298
298
299 The HDF5 file contains by default two groups Data and Metadata where
299 The HDF5 file contains by default two groups Data and Metadata where
300 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
300 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
301 parameters, data attributes are normaly time dependent where the metadata
301 parameters, data attributes are normaly time dependent where the metadata
302 are not.
302 are not.
303 It is possible to customize the structure of the HDF5 file with the
303 It is possible to customize the structure of the HDF5 file with the
304 optional description parameter see the examples.
304 optional description parameter see the examples.
305
305
306 Parameters:
306 Parameters:
307 -----------
307 -----------
308 path : str
308 path : str
309 Path where files will be saved.
309 Path where files will be saved.
310 blocksPerFile : int
310 blocksPerFile : int
311 Number of blocks per file
311 Number of blocks per file
312 metadataList : list
312 metadataList : list
313 List of the dataOut attributes that will be saved as metadata
313 List of the dataOut attributes that will be saved as metadata
314 dataList : int
314 dataList : int
315 List of the dataOut attributes that will be saved as data
315 List of the dataOut attributes that will be saved as data
316 setType : bool
316 setType : bool
317 If True the name of the files corresponds to the timestamp of the data
317 If True the name of the files corresponds to the timestamp of the data
318 description : dict, optional
318 description : dict, optional
319 Dictionary with the desired description of the HDF5 file
319 Dictionary with the desired description of the HDF5 file
320
320
321 Examples
321 Examples
322 --------
322 --------
323
323
324 desc = {
324 desc = {
325 'data_output': {'winds': ['z', 'w', 'v']},
325 'data_output': {'winds': ['z', 'w', 'v']},
326 'utctime': 'timestamps',
326 'utctime': 'timestamps',
327 'heightList': 'heights'
327 'heightList': 'heights'
328 }
328 }
329 desc = {
329 desc = {
330 'data_output': ['z', 'w', 'v'],
330 'data_output': ['z', 'w', 'v'],
331 'utctime': 'timestamps',
331 'utctime': 'timestamps',
332 'heightList': 'heights'
332 'heightList': 'heights'
333 }
333 }
334 desc = {
334 desc = {
335 'Data': {
335 'Data': {
336 'data_output': 'winds',
336 'data_output': 'winds',
337 'utctime': 'timestamps'
337 'utctime': 'timestamps'
338 },
338 },
339 'Metadata': {
339 'Metadata': {
340 'heightList': 'heights'
340 'heightList': 'heights'
341 }
341 }
342 }
342 }
343
343
344 writer = proc_unit.addOperation(name='HDFWriter')
344 writer = proc_unit.addOperation(name='HDFWriter')
345 writer.addParameter(name='path', value='/path/to/file')
345 writer.addParameter(name='path', value='/path/to/file')
346 writer.addParameter(name='blocksPerFile', value='32')
346 writer.addParameter(name='blocksPerFile', value='32')
347 writer.addParameter(name='metadataList', value='heightList,timeZone')
347 writer.addParameter(name='metadataList', value='heightList,timeZone')
348 writer.addParameter(name='dataList',value='data_output,utctime')
348 writer.addParameter(name='dataList',value='data_output,utctime')
349 # writer.addParameter(name='description',value=json.dumps(desc))
349 # writer.addParameter(name='description',value=json.dumps(desc))
350
350
351 """
351 """
352
352
353 ext = ".hdf5"
353 ext = ".hdf5"
354 optchar = "D"
354 optchar = "D"
355 filename = None
355 filename = None
356 path = None
356 path = None
357 setFile = None
357 setFile = None
358 fp = None
358 fp = None
359 firsttime = True
359 firsttime = True
360 #Configurations
360 #Configurations
361 blocksPerFile = None
361 blocksPerFile = None
362 blockIndex = None
362 blockIndex = None
363 dataOut = None
363 dataOut = None
364 #Data Arrays
364 #Data Arrays
365 dataList = None
365 dataList = None
366 metadataList = None
366 metadataList = None
367 currentDay = None
367 currentDay = None
368 lastTime = None
368 lastTime = None
369 last_Azipos = None
369 last_Azipos = None
370 last_Elepos = None
370 last_Elepos = None
371 mode = None
371 mode = None
372 #-----------------------
372 #-----------------------
373 Typename = None
373 Typename = None
374 mask = False
374 mask = False
375
375
376 def __init__(self):
376 def __init__(self):
377
377
378 Operation.__init__(self)
378 Operation.__init__(self)
379 return
379 return
380
380
381 def set_kwargs(self, **kwargs):
381 def set_kwargs(self, **kwargs):
382
382
383 for key, value in kwargs.items():
383 for key, value in kwargs.items():
384 setattr(self, key, value)
384 setattr(self, key, value)
385
385
386 def set_kwargs_obj(self,obj, **kwargs):
386 def set_kwargs_obj(self,obj, **kwargs):
387
387
388 for key, value in kwargs.items():
388 for key, value in kwargs.items():
389 setattr(obj, key, value)
389 setattr(obj, key, value)
390
390
391 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None,type_data=None, localtime=True, **kwargs):
391 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None,type_data=None, localtime=True, **kwargs):
392 self.path = path
392 self.path = path
393 self.blocksPerFile = blocksPerFile
393 self.blocksPerFile = blocksPerFile
394 self.metadataList = metadataList
394 self.metadataList = metadataList
395 self.dataList = [s.strip() for s in dataList]
395 self.dataList = [s.strip() for s in dataList]
396 self.setType = setType
396 self.setType = setType
397 if self.setType == "weather":
397 if self.setType == "weather":
398 self.set_kwargs(**kwargs)
398 self.set_kwargs(**kwargs)
399 self.set_kwargs_obj(self.dataOut,**kwargs)
399 self.set_kwargs_obj(self.dataOut,**kwargs)
400 self.weather_vars = {
400 self.weather_vars = {
401 'S' : 0,
401 'S' : 0,
402 'V' : 1,
402 'V' : 1,
403 'W' : 2,
403 'W' : 2,
404 'SNR' : 3,
404 'SNR' : 3,
405 'Z' : 4,
405 'Z' : 4,
406 'D' : 5,
406 'D' : 5,
407 'P' : 6,
407 'P' : 6,
408 'R' : 7,
408 'R' : 7,
409 }
409 }
410
410
411 if localtime:
411 if localtime:
412 self.getDateTime = datetime.datetime.fromtimestamp
412 self.getDateTime = datetime.datetime.fromtimestamp
413 else:
413 else:
414 self.getDateTime = datetime.datetime.utcfromtimestamp
414 self.getDateTime = datetime.datetime.utcfromtimestamp
415
415
416 self.description = description
416 self.description = description
417 self.type_data=type_data
417 self.type_data=type_data
418
418
419 if self.metadataList is None:
419 if self.metadataList is None:
420 self.metadataList = self.dataOut.metadata_list
420 self.metadataList = self.dataOut.metadata_list
421
421
422 dsList = []
422 dsList = []
423
423
424 for i in range(len(self.dataList)):
424 for i in range(len(self.dataList)):
425 dsDict = {}
425 dsDict = {}
426 if hasattr(self.dataOut, self.dataList[i]):
426 if hasattr(self.dataOut, self.dataList[i]):
427 dataAux = getattr(self.dataOut, self.dataList[i])
427 dataAux = getattr(self.dataOut, self.dataList[i])
428 if self.setType == 'weather' and self.dataList[i] == 'data_param':
428 if self.setType == 'weather' and self.dataList[i] == 'data_param':
429 dataAux = dataAux[:,self.weather_vars[self.weather_var],:]
429 dataAux = dataAux[:,self.weather_vars[self.weather_var],:]
430 dsDict['variable'] = self.dataList[i]
430 dsDict['variable'] = self.dataList[i]
431 else:
431 else:
432 log.warning('Attribute {} not found in dataOut'.format(self.dataList[i]), self.name)
432 log.warning('Attribute {} not found in dataOut'.format(self.dataList[i]), self.name)
433 continue
433 continue
434
434
435 if dataAux is None:
435 if dataAux is None:
436 continue
436 continue
437 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
437 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
438 dsDict['nDim'] = 0
438 dsDict['nDim'] = 0
439 else:
439 else:
440 dsDict['nDim'] = len(dataAux.shape)
440 dsDict['nDim'] = len(dataAux.shape)
441 dsDict['shape'] = dataAux.shape
441 dsDict['shape'] = dataAux.shape
442 dsDict['dsNumber'] = dataAux.shape[0]
442 dsDict['dsNumber'] = dataAux.shape[0]
443 dsDict['dtype'] = dataAux.dtype
443 dsDict['dtype'] = dataAux.dtype
444 dsList.append(dsDict)
444 dsList.append(dsDict)
445
445
446 self.dsList = dsList
446 self.dsList = dsList
447 self.currentDay = self.dataOut.datatime.date()
447 self.currentDay = self.dataOut.datatime.date()
448
448
449 def timeFlag(self):
449 def timeFlag(self):
450 currentTime = self.dataOut.utctime
450 currentTime = self.dataOut.utctime
451 dt = self.getDateTime(currentTime)
451 dt = self.getDateTime(currentTime)
452
452
453 dataDay = int(dt.strftime('%j'))
453 dataDay = int(dt.strftime('%j'))
454
454
455 if self.lastTime is None:
455 if self.lastTime is None:
456 self.lastTime = currentTime
456 self.lastTime = currentTime
457 self.currentDay = dataDay
457 self.currentDay = dataDay
458 return False
458 return False
459
459
460 timeDiff = currentTime - self.lastTime
460 timeDiff = currentTime - self.lastTime
461
461
462 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
462 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
463 if dataDay != self.currentDay:
463 if dataDay != self.currentDay:
464 self.currentDay = dataDay
464 self.currentDay = dataDay
465 return True
465 return True
466 elif timeDiff > 3*60*60:
466 elif timeDiff > 3*60*60:
467 self.lastTime = currentTime
467 self.lastTime = currentTime
468 return True
468 return True
469 else:
469 else:
470 self.lastTime = currentTime
470 self.lastTime = currentTime
471 return False
471 return False
472
472
473 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
473 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
474 dataList=[], setType=None, description={}, mode= None,
474 dataList=[], setType=None, description={}, mode= None,
475 type_data=None, Reset = False, localtime=True, **kwargs):
475 type_data=None, Reset = False, localtime=True, **kwargs):
476
476
477 if Reset:
477 if Reset:
478 self.isConfig = False
478 self.isConfig = False
479 self.closeFile()
479 self.closeFile()
480 self.lastTime = None
480 self.lastTime = None
481 self.blockIndex = 0
481 self.blockIndex = 0
482
482
483 self.dataOut = dataOut
483 self.dataOut = dataOut
484 self.mode = mode
484 self.mode = mode
485
485
486 if not(self.isConfig):
486 if not(self.isConfig):
487 self.setup(path=path, blocksPerFile=blocksPerFile,
487 self.setup(path=path, blocksPerFile=blocksPerFile,
488 metadataList=metadataList, dataList=dataList,
488 metadataList=metadataList, dataList=dataList,
489 setType=setType, description=description,type_data=type_data,
489 setType=setType, description=description,type_data=type_data,
490 localtime=localtime, **kwargs)
490 localtime=localtime, **kwargs)
491
491
492 self.isConfig = True
492 self.isConfig = True
493 self.setNextFile()
493 self.setNextFile()
494
494
495 self.putData()
495 self.putData()
496 return
496 return
497
497
498 def setNextFile(self):
498 def setNextFile(self):
499
499
500 ext = self.ext
500 ext = self.ext
501 path = self.path
501 path = self.path
502 setFile = self.setFile
502 setFile = self.setFile
503
503
504 dt = self.getDateTime(self.dataOut.utctime)
504 dt = self.getDateTime(self.dataOut.utctime)
505
505
506 if self.setType == 'weather':
506 if self.setType == 'weather':
507 subfolder = dt.strftime('%Y-%m-%dT%H-00-00')
507 subfolder = dt.strftime('%Y-%m-%dT%H-00-00')
508 subfolder = ''
508 else:
509 else:
509 subfolder = dt.strftime('d%Y%j')
510 subfolder = dt.strftime('d%Y%j')
510
511
511 fullpath = os.path.join(path, subfolder)
512 fullpath = os.path.join(path, subfolder)
512
513
513 if os.path.exists(fullpath):
514 if os.path.exists(fullpath):
514 filesList = os.listdir(fullpath)
515 filesList = os.listdir(fullpath)
515 filesList = [k for k in filesList if k.startswith(self.optchar)]
516 filesList = [k for k in filesList if k.startswith(self.optchar)]
516 if len( filesList ) > 0:
517 if len( filesList ) > 0:
517 filesList = sorted(filesList, key=str.lower)
518 filesList = sorted(filesList, key=str.lower)
518 filen = filesList[-1]
519 filen = filesList[-1]
519 # el filename debera tener el siguiente formato
520 # el filename debera tener el siguiente formato
520 # 0 1234 567 89A BCDE (hex)
521 # 0 1234 567 89A BCDE (hex)
521 # x YYYY DDD SSS .ext
522 # x YYYY DDD SSS .ext
522 if isNumber(filen[8:11]):
523 if isNumber(filen[8:11]):
523 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
524 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
524 else:
525 else:
525 setFile = -1
526 setFile = -1
526 else:
527 else:
527 setFile = -1 #inicializo mi contador de seteo
528 setFile = -1 #inicializo mi contador de seteo
528 else:
529 else:
529 os.makedirs(fullpath)
530 os.makedirs(fullpath)
530 setFile = -1 #inicializo mi contador de seteo
531 setFile = -1 #inicializo mi contador de seteo
531
532
532 if self.setType is None:
533 if self.setType is None:
533 setFile += 1
534 setFile += 1
534 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
535 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
535 dt.year,
536 dt.year,
536 int(dt.strftime('%j')),
537 int(dt.strftime('%j')),
537 setFile,
538 setFile,
538 ext )
539 ext )
539 elif self.setType == "weather":
540 elif self.setType == "weather":
540
541
541 #SOPHY_20200505_140215_E10.0_Z.h5
542 #SOPHY_20200505_140215_E10.0_Z.h5
542 #SOPHY_20200505_140215_A40.0_Z.h5
543 #SOPHY_20200505_140215_A40.0_Z.h5
543 if self.dataOut.flagMode == 1: #'AZI' #PPI
544 if self.dataOut.flagMode == 1: #'AZI' #PPI
544 ang_type = 'E'
545 ang_type = 'EL'
546 mode_type = 'PPI'
545 len_aux = int(self.dataOut.data_ele.shape[0]/4)
547 len_aux = int(self.dataOut.data_ele.shape[0]/4)
546 mean = numpy.mean(self.dataOut.data_ele[len_aux:-len_aux])
548 mean = numpy.mean(self.dataOut.data_ele[len_aux:-len_aux])
547 ang_ = round(mean,1)
549 ang_ = round(mean,1)
548 elif self.dataOut.flagMode == 0: #'ELE' #RHI
550 elif self.dataOut.flagMode == 0: #'ELE' #RHI
549 ang_type = 'A'
551 ang_type = 'AZ'
552 mode_type = 'RHI'
550 len_aux = int(self.dataOut.data_azi.shape[0]/4)
553 len_aux = int(self.dataOut.data_azi.shape[0]/4)
551 mean = numpy.mean(self.dataOut.data_azi[len_aux:-len_aux])
554 mean = numpy.mean(self.dataOut.data_azi[len_aux:-len_aux])
552 ang_ = round(mean,1)
555 ang_ = round(mean,1)
553
556
554 file = '%s_%2.2d%2.2d%2.2d_%2.2d%2.2d%2.2d_%s%2.1f_%s%s' % (
557 file = '%s_%2.2d%2.2d%2.2d_%2.2d%2.2d%2.2d_%s%2.1f_%s%s' % (
555 'SOPHY',
558 'SOPHY',
556 dt.year,
559 dt.year,
557 dt.month,
560 dt.month,
558 dt.day,
561 dt.day,
559 dt.hour,
562 dt.hour,
560 dt.minute,
563 dt.minute,
561 dt.second,
564 dt.second,
562 ang_type,
565 ang_type[0],
563 ang_,
566 ang_,
564 self.weather_var,
567 self.weather_var,
565 ext )
568 ext )
566
569 subfolder = '{}_{}_{}_{:2.1f}'.format(self.weather_var, mode_type, ang_type, ang_)
570 fullpath = os.path.join(path, subfolder)
571 if not os.path.exists(fullpath):
572 os.makedirs(fullpath)
567 else:
573 else:
568 setFile = dt.hour*60+dt.minute
574 setFile = dt.hour*60+dt.minute
569 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
575 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
570 dt.year,
576 dt.year,
571 int(dt.strftime('%j')),
577 int(dt.strftime('%j')),
572 setFile,
578 setFile,
573 ext )
579 ext )
574
580
575 self.filename = os.path.join( path, subfolder, file )
581 self.filename = os.path.join( path, subfolder, file )
576
582
577 self.fp = h5py.File(self.filename, 'w')
583 self.fp = h5py.File(self.filename, 'w')
578 #write metadata
584 #write metadata
579 self.writeMetadata(self.fp)
585 self.writeMetadata(self.fp)
580 #Write data
586 #Write data
581 self.writeData(self.fp)
587 self.writeData(self.fp)
582
588
583 def getLabel(self, name, x=None):
589 def getLabel(self, name, x=None):
584
590
585 if x is None:
591 if x is None:
586 if 'Data' in self.description:
592 if 'Data' in self.description:
587 data = self.description['Data']
593 data = self.description['Data']
588 if 'Metadata' in self.description:
594 if 'Metadata' in self.description:
589 data.update(self.description['Metadata'])
595 data.update(self.description['Metadata'])
590 else:
596 else:
591 data = self.description
597 data = self.description
592 if name in data:
598 if name in data:
593 if isinstance(data[name], str):
599 if isinstance(data[name], str):
594 return data[name]
600 return data[name]
595 elif isinstance(data[name], list):
601 elif isinstance(data[name], list):
596 return None
602 return None
597 elif isinstance(data[name], dict):
603 elif isinstance(data[name], dict):
598 for key, value in data[name].items():
604 for key, value in data[name].items():
599 return key
605 return key
600 return name
606 return name
601 else:
607 else:
602 if 'Data' in self.description:
608 if 'Data' in self.description:
603 data = self.description['Data']
609 data = self.description['Data']
604 if 'Metadata' in self.description:
610 if 'Metadata' in self.description:
605 data.update(self.description['Metadata'])
611 data.update(self.description['Metadata'])
606 else:
612 else:
607 data = self.description
613 data = self.description
608 if name in data:
614 if name in data:
609 if isinstance(data[name], list):
615 if isinstance(data[name], list):
610 return data[name][x]
616 return data[name][x]
611 elif isinstance(data[name], dict):
617 elif isinstance(data[name], dict):
612 for key, value in data[name].items():
618 for key, value in data[name].items():
613 return value[x]
619 return value[x]
614 if 'cspc' in name:
620 if 'cspc' in name:
615 return 'pair{:02d}'.format(x)
621 return 'pair{:02d}'.format(x)
616 else:
622 else:
617 return 'channel{:02d}'.format(x)
623 return 'channel{:02d}'.format(x)
618
624
619 def writeMetadata(self, fp):
625 def writeMetadata(self, fp):
620
626
621 if self.description:
627 if self.description:
622 if 'Metadata' in self.description:
628 if 'Metadata' in self.description:
623 grp = fp.create_group('Metadata')
629 grp = fp.create_group('Metadata')
624 else:
630 else:
625 grp = fp
631 grp = fp
626 else:
632 else:
627 grp = fp.create_group('Metadata')
633 grp = fp.create_group('Metadata')
628
634
629 for i in range(len(self.metadataList)):
635 for i in range(len(self.metadataList)):
630 if not hasattr(self.dataOut, self.metadataList[i]):
636 if not hasattr(self.dataOut, self.metadataList[i]):
631 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
637 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
632 continue
638 continue
633 value = getattr(self.dataOut, self.metadataList[i])
639 value = getattr(self.dataOut, self.metadataList[i])
634 if isinstance(value, bool):
640 if isinstance(value, bool):
635 if value is True:
641 if value is True:
636 value = 1
642 value = 1
637 else:
643 else:
638 value = 0
644 value = 0
639 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
645 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
640 return
646 return
641
647
642 def writeData(self, fp):
648 def writeData(self, fp):
643
649
644 if self.description:
650 if self.description:
645 if 'Data' in self.description:
651 if 'Data' in self.description:
646 grp = fp.create_group('Data')
652 grp = fp.create_group('Data')
647 else:
653 else:
648 grp = fp
654 grp = fp
649 else:
655 else:
650 grp = fp.create_group('Data')
656 grp = fp.create_group('Data')
651
657
652 dtsets = []
658 dtsets = []
653 data = []
659 data = []
654
660
655 for dsInfo in self.dsList:
661 for dsInfo in self.dsList:
656
662
657 if dsInfo['nDim'] == 0:
663 if dsInfo['nDim'] == 0:
658 ds = grp.create_dataset(
664 ds = grp.create_dataset(
659 self.getLabel(dsInfo['variable']),
665 self.getLabel(dsInfo['variable']),
660 (self.blocksPerFile, ),
666 (self.blocksPerFile, ),
661 chunks=True,
667 chunks=True,
662 dtype=numpy.float64)
668 dtype=numpy.float64)
663 dtsets.append(ds)
669 dtsets.append(ds)
664 data.append((dsInfo['variable'], -1))
670 data.append((dsInfo['variable'], -1))
665 else:
671 else:
666 label = self.getLabel(dsInfo['variable'])
672 label = self.getLabel(dsInfo['variable'])
667 if label is not None:
673 if label is not None:
668 sgrp = grp.create_group(label)
674 sgrp = grp.create_group(label)
669 else:
675 else:
670 sgrp = grp
676 sgrp = grp
671 if self.blocksPerFile == 1:
677 if self.blocksPerFile == 1:
672 shape = dsInfo['shape'][1:]
678 shape = dsInfo['shape'][1:]
673 else:
679 else:
674 shape = (self.blocksPerFile, ) + dsInfo['shape'][1:]
680 shape = (self.blocksPerFile, ) + dsInfo['shape'][1:]
675 for i in range(dsInfo['dsNumber']):
681 for i in range(dsInfo['dsNumber']):
676 ds = sgrp.create_dataset(
682 ds = sgrp.create_dataset(
677 self.getLabel(dsInfo['variable'], i),
683 self.getLabel(dsInfo['variable'], i),
678 shape,
684 shape,
679 chunks=True,
685 chunks=True,
680 dtype=dsInfo['dtype'],
686 dtype=dsInfo['dtype'],
681 compression='gzip',
687 compression='gzip',
682 )
688 )
683 dtsets.append(ds)
689 dtsets.append(ds)
684 data.append((dsInfo['variable'], i))
690 data.append((dsInfo['variable'], i))
685 fp.flush()
691 fp.flush()
686
692
687 log.log('Creating file: {}'.format(fp.filename), self.name)
693 log.log('Creating file: {}'.format(fp.filename), self.name)
688
694
689 self.ds = dtsets
695 self.ds = dtsets
690 self.data = data
696 self.data = data
691 self.firsttime = True
697 self.firsttime = True
692 self.blockIndex = 0
698 self.blockIndex = 0
693 return
699 return
694
700
695 def putData(self):
701 def putData(self):
696
702
697 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
703 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
698 self.closeFile()
704 self.closeFile()
699 self.setNextFile()
705 self.setNextFile()
700
706
701 for i, ds in enumerate(self.ds):
707 for i, ds in enumerate(self.ds):
702 attr, ch = self.data[i]
708 attr, ch = self.data[i]
703 if ch == -1:
709 if ch == -1:
704 ds[self.blockIndex] = getattr(self.dataOut, attr)
710 ds[self.blockIndex] = getattr(self.dataOut, attr)
705 else:
711 else:
706 if self.blocksPerFile == 1:
712 if self.blocksPerFile == 1:
707 mask = self.dataOut.data_param[:,3,:][ch] < self.mask
713 mask = self.dataOut.data_param[:,3,:][ch] < self.mask
708 tmp = getattr(self.dataOut, attr)[:,self.weather_vars[self.weather_var],:][ch]
714 tmp = getattr(self.dataOut, attr)[:,self.weather_vars[self.weather_var],:][ch]
709 if self.mask:
715 if self.mask:
710 tmp[mask] = numpy.nan
716 tmp[mask] = numpy.nan
711 ds[:] = tmp
717 ds[:] = tmp
712 else:
718 else:
713 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
719 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
714
720
715 self.fp.flush()
721 self.fp.flush()
716 self.blockIndex += 1
722 self.blockIndex += 1
717 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
723 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
718
724
719 return
725 return
720
726
721 def closeFile(self):
727 def closeFile(self):
722
728
723 if self.blockIndex != self.blocksPerFile:
729 if self.blockIndex != self.blocksPerFile:
724 for ds in self.ds:
730 for ds in self.ds:
725 ds.resize(self.blockIndex, axis=0)
731 ds.resize(self.blockIndex, axis=0)
726
732
727 if self.fp:
733 if self.fp:
728 self.fp.flush()
734 self.fp.flush()
729 self.fp.close()
735 self.fp.close()
730
736
731 def close(self):
737 def close(self):
732
738
733 self.closeFile()
739 self.closeFile()
Show file before | Show file after | Hide comments
@@ -1,4531 +1,4549
1
1
2 import os
2 import os
3 import time
3 import time
4 import math
4 import math
5
5
6 import re
6 import re
7 import datetime
7 import datetime
8 import copy
8 import copy
9 import sys
9 import sys
10 import importlib
10 import importlib
11 import itertools
11 import itertools
12
12
13 from multiprocessing import Pool, TimeoutError
13 from multiprocessing import Pool, TimeoutError
14 from multiprocessing.pool import ThreadPool
14 from multiprocessing.pool import ThreadPool
15 import numpy
15 import numpy
16 import glob
16 import glob
17 import scipy
17 import scipy
18 import h5py
18 import h5py
19 from scipy.optimize import fmin_l_bfgs_b #optimize with bounds on state papameters
19 from scipy.optimize import fmin_l_bfgs_b #optimize with bounds on state papameters
20 from .jroproc_base import ProcessingUnit, Operation, MPDecorator
20 from .jroproc_base import ProcessingUnit, Operation, MPDecorator
21 from schainpy.model.data.jrodata import Parameters, hildebrand_sekhon
21 from schainpy.model.data.jrodata import Parameters, hildebrand_sekhon
22 from scipy import asarray as ar,exp
22 from scipy import asarray as ar,exp
23 from scipy.optimize import curve_fit
23 from scipy.optimize import curve_fit
24 from schainpy.utils import log
24 from schainpy.utils import log
25 import schainpy.admin
25 import schainpy.admin
26 import warnings
26 import warnings
27 from scipy import optimize, interpolate, signal, stats, ndimage
27 from scipy import optimize, interpolate, signal, stats, ndimage
28 from scipy.optimize.optimize import OptimizeWarning
28 from scipy.optimize.optimize import OptimizeWarning
29 warnings.filterwarnings('ignore')
29 warnings.filterwarnings('ignore')
30
30
31
31
32 SPEED_OF_LIGHT = 299792458
32 SPEED_OF_LIGHT = 299792458
33
33
34 '''solving pickling issue'''
34 '''solving pickling issue'''
35
35
36 def _pickle_method(method):
36 def _pickle_method(method):
37 func_name = method.__func__.__name__
37 func_name = method.__func__.__name__
38 obj = method.__self__
38 obj = method.__self__
39 cls = method.__self__.__class__
39 cls = method.__self__.__class__
40 return _unpickle_method, (func_name, obj, cls)
40 return _unpickle_method, (func_name, obj, cls)
41
41
42 def _unpickle_method(func_name, obj, cls):
42 def _unpickle_method(func_name, obj, cls):
43 for cls in cls.mro():
43 for cls in cls.mro():
44 try:
44 try:
45 func = cls.__dict__[func_name]
45 func = cls.__dict__[func_name]
46 except KeyError:
46 except KeyError:
47 pass
47 pass
48 else:
48 else:
49 break
49 break
50 return func.__get__(obj, cls)
50 return func.__get__(obj, cls)
51
51
52 def isNumber(str):
52 def isNumber(str):
53 try:
53 try:
54 float(str)
54 float(str)
55 return True
55 return True
56 except:
56 except:
57 return False
57 return False
58
58
59 class ParametersProc(ProcessingUnit):
59 class ParametersProc(ProcessingUnit):
60
60
61 METHODS = {}
61 METHODS = {}
62 nSeconds = None
62 nSeconds = None
63
63
64 def __init__(self):
64 def __init__(self):
65 ProcessingUnit.__init__(self)
65 ProcessingUnit.__init__(self)
66
66
67 # self.objectDict = {}
67 # self.objectDict = {}
68 self.buffer = None
68 self.buffer = None
69 self.firstdatatime = None
69 self.firstdatatime = None
70 self.profIndex = 0
70 self.profIndex = 0
71 self.dataOut = Parameters()
71 self.dataOut = Parameters()
72 self.setupReq = False #Agregar a todas las unidades de proc
72 self.setupReq = False #Agregar a todas las unidades de proc
73
73
74 def __updateObjFromInput(self):
74 def __updateObjFromInput(self):
75
75
76 self.dataOut.inputUnit = self.dataIn.type
76 self.dataOut.inputUnit = self.dataIn.type
77
77
78 self.dataOut.timeZone = self.dataIn.timeZone
78 self.dataOut.timeZone = self.dataIn.timeZone
79 self.dataOut.dstFlag = self.dataIn.dstFlag
79 self.dataOut.dstFlag = self.dataIn.dstFlag
80 self.dataOut.errorCount = self.dataIn.errorCount
80 self.dataOut.errorCount = self.dataIn.errorCount
81 self.dataOut.useLocalTime = self.dataIn.useLocalTime
81 self.dataOut.useLocalTime = self.dataIn.useLocalTime
82
82
83 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
83 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
84 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
84 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
85 self.dataOut.channelList = self.dataIn.channelList
85 self.dataOut.channelList = self.dataIn.channelList
86 self.dataOut.heightList = self.dataIn.heightList
86 self.dataOut.heightList = self.dataIn.heightList
87 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
87 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
88 # self.dataOut.nHeights = self.dataIn.nHeights
88 # self.dataOut.nHeights = self.dataIn.nHeights
89 # self.dataOut.nChannels = self.dataIn.nChannels
89 # self.dataOut.nChannels = self.dataIn.nChannels
90 # self.dataOut.nBaud = self.dataIn.nBaud
90 # self.dataOut.nBaud = self.dataIn.nBaud
91 # self.dataOut.nCode = self.dataIn.nCode
91 # self.dataOut.nCode = self.dataIn.nCode
92 # self.dataOut.code = self.dataIn.code
92 # self.dataOut.code = self.dataIn.code
93 # self.dataOut.nProfiles = self.dataOut.nFFTPoints
93 # self.dataOut.nProfiles = self.dataOut.nFFTPoints
94 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
94 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
95 # self.dataOut.utctime = self.firstdatatime
95 # self.dataOut.utctime = self.firstdatatime
96 self.dataOut.utctime = self.dataIn.utctime
96 self.dataOut.utctime = self.dataIn.utctime
97 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
97 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
98 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
98 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
99 self.dataOut.nCohInt = self.dataIn.nCohInt
99 self.dataOut.nCohInt = self.dataIn.nCohInt
100 # self.dataOut.nIncohInt = 1
100 # self.dataOut.nIncohInt = 1
101 # self.dataOut.ippSeconds = self.dataIn.ippSeconds
101 # self.dataOut.ippSeconds = self.dataIn.ippSeconds
102 # self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
102 # self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
103 self.dataOut.timeInterval1 = self.dataIn.timeInterval
103 self.dataOut.timeInterval1 = self.dataIn.timeInterval
104 self.dataOut.heightList = self.dataIn.heightList
104 self.dataOut.heightList = self.dataIn.heightList
105 self.dataOut.frequency = self.dataIn.frequency
105 self.dataOut.frequency = self.dataIn.frequency
106 # self.dataOut.noise = self.dataIn.noise
106 # self.dataOut.noise = self.dataIn.noise
107 self.dataOut.runNextUnit = self.dataIn.runNextUnit
107 self.dataOut.runNextUnit = self.dataIn.runNextUnit
108 self.dataOut.h0 = self.dataIn.h0
108 self.dataOut.h0 = self.dataIn.h0
109
109
110 def run(self, runNextUnit = 0):
110 def run(self, runNextUnit = 0):
111
111
112 self.dataIn.runNextUnit = runNextUnit
112 self.dataIn.runNextUnit = runNextUnit
113 #print("HOLA MUNDO SOY YO")
113 #print("HOLA MUNDO SOY YO")
114 #---------------------- Voltage Data ---------------------------
114 #---------------------- Voltage Data ---------------------------
115
115
116 if self.dataIn.type == "Voltage":
116 if self.dataIn.type == "Voltage":
117
117
118 self.__updateObjFromInput()
118 self.__updateObjFromInput()
119 self.dataOut.data_pre = self.dataIn.data.copy()
119 self.dataOut.data_pre = self.dataIn.data.copy()
120 self.dataOut.flagNoData = False
120 self.dataOut.flagNoData = False
121 self.dataOut.utctimeInit = self.dataIn.utctime
121 self.dataOut.utctimeInit = self.dataIn.utctime
122 self.dataOut.paramInterval = self.dataIn.nProfiles*self.dataIn.nCohInt*self.dataIn.ippSeconds
122 self.dataOut.paramInterval = self.dataIn.nProfiles*self.dataIn.nCohInt*self.dataIn.ippSeconds
123
123
124 if hasattr(self.dataIn, 'flagDataAsBlock'):
124 if hasattr(self.dataIn, 'flagDataAsBlock'):
125 self.dataOut.flagDataAsBlock = self.dataIn.flagDataAsBlock
125 self.dataOut.flagDataAsBlock = self.dataIn.flagDataAsBlock
126
126
127 if hasattr(self.dataIn, 'profileIndex'):
127 if hasattr(self.dataIn, 'profileIndex'):
128 self.dataOut.profileIndex = self.dataIn.profileIndex
128 self.dataOut.profileIndex = self.dataIn.profileIndex
129
129
130 if hasattr(self.dataIn, 'dataPP_POW'):
130 if hasattr(self.dataIn, 'dataPP_POW'):
131 self.dataOut.dataPP_POW = self.dataIn.dataPP_POW
131 self.dataOut.dataPP_POW = self.dataIn.dataPP_POW
132
132
133 if hasattr(self.dataIn, 'dataPP_POWER'):
133 if hasattr(self.dataIn, 'dataPP_POWER'):
134 self.dataOut.dataPP_POWER = self.dataIn.dataPP_POWER
134 self.dataOut.dataPP_POWER = self.dataIn.dataPP_POWER
135
135
136 if hasattr(self.dataIn, 'dataPP_DOP'):
136 if hasattr(self.dataIn, 'dataPP_DOP'):
137 self.dataOut.dataPP_DOP = self.dataIn.dataPP_DOP
137 self.dataOut.dataPP_DOP = self.dataIn.dataPP_DOP
138
138
139 if hasattr(self.dataIn, 'dataPP_SNR'):
139 if hasattr(self.dataIn, 'dataPP_SNR'):
140 self.dataOut.dataPP_SNR = self.dataIn.dataPP_SNR
140 self.dataOut.dataPP_SNR = self.dataIn.dataPP_SNR
141
141
142 if hasattr(self.dataIn, 'dataPP_WIDTH'):
142 if hasattr(self.dataIn, 'dataPP_WIDTH'):
143 self.dataOut.dataPP_WIDTH = self.dataIn.dataPP_WIDTH
143 self.dataOut.dataPP_WIDTH = self.dataIn.dataPP_WIDTH
144
144
145 if hasattr(self.dataIn, 'dataPP_CCF'):
145 if hasattr(self.dataIn, 'dataPP_CCF'):
146 self.dataOut.dataPP_CCF = self.dataIn.dataPP_CCF
146 self.dataOut.dataPP_CCF = self.dataIn.dataPP_CCF
147
147
148 if hasattr(self.dataIn, 'dataPP_NOISE'):
148 if hasattr(self.dataIn, 'dataPP_NOISE'):
149 self.dataOut.dataPP_NOISE = self.dataIn.dataPP_NOISE
149 self.dataOut.dataPP_NOISE = self.dataIn.dataPP_NOISE
150
150
151 if hasattr(self.dataIn, 'flagAskMode'):
151 if hasattr(self.dataIn, 'flagAskMode'):
152 self.dataOut.flagAskMode = self.dataIn.flagAskMode
152 self.dataOut.flagAskMode = self.dataIn.flagAskMode
153
153
154 return
154 return
155
155
156 #---------------------- Spectra Data ---------------------------
156 #---------------------- Spectra Data ---------------------------
157
157
158 if self.dataIn.type == "Spectra":
158 if self.dataIn.type == "Spectra":
159 #print("que paso en spectra")
159 #print("que paso en spectra")
160 self.dataOut.data_pre = [self.dataIn.data_spc, self.dataIn.data_cspc]
160 self.dataOut.data_pre = [self.dataIn.data_spc, self.dataIn.data_cspc]
161 self.dataOut.data_spc = self.dataIn.data_spc
161 self.dataOut.data_spc = self.dataIn.data_spc
162 self.dataOut.data_cspc = self.dataIn.data_cspc
162 self.dataOut.data_cspc = self.dataIn.data_cspc
163 self.dataOut.nProfiles = self.dataIn.nProfiles
163 self.dataOut.nProfiles = self.dataIn.nProfiles
164 self.dataOut.nIncohInt = self.dataIn.nIncohInt
164 self.dataOut.nIncohInt = self.dataIn.nIncohInt
165 self.dataOut.nFFTPoints = self.dataIn.nFFTPoints
165 self.dataOut.nFFTPoints = self.dataIn.nFFTPoints
166 self.dataOut.ippFactor = self.dataIn.ippFactor
166 self.dataOut.ippFactor = self.dataIn.ippFactor
167 self.dataOut.abscissaList = self.dataIn.getVelRange(1)
167 self.dataOut.abscissaList = self.dataIn.getVelRange(1)
168 self.dataOut.spc_noise = self.dataIn.getNoise()
168 self.dataOut.spc_noise = self.dataIn.getNoise()
169 self.dataOut.spc_range = (self.dataIn.getFreqRange(1) , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1))
169 self.dataOut.spc_range = (self.dataIn.getFreqRange(1) , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1))
170 # self.dataOut.normFactor = self.dataIn.normFactor
170 # self.dataOut.normFactor = self.dataIn.normFactor
171 self.dataOut.pairsList = self.dataIn.pairsList
171 self.dataOut.pairsList = self.dataIn.pairsList
172 self.dataOut.groupList = self.dataIn.pairsList
172 self.dataOut.groupList = self.dataIn.pairsList
173 self.dataOut.flagNoData = False
173 self.dataOut.flagNoData = False
174
174
175 if hasattr(self.dataIn, 'flagDataAsBlock'):
175 if hasattr(self.dataIn, 'flagDataAsBlock'):
176 self.dataOut.flagDataAsBlock = self.dataIn.flagDataAsBlock
176 self.dataOut.flagDataAsBlock = self.dataIn.flagDataAsBlock
177
177
178 if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels
178 if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels
179 self.dataOut.ChanDist = self.dataIn.ChanDist
179 self.dataOut.ChanDist = self.dataIn.ChanDist
180 else: self.dataOut.ChanDist = None
180 else: self.dataOut.ChanDist = None
181
181
182 #if hasattr(self.dataIn, 'VelRange'): #Velocities range
182 #if hasattr(self.dataIn, 'VelRange'): #Velocities range
183 # self.dataOut.VelRange = self.dataIn.VelRange
183 # self.dataOut.VelRange = self.dataIn.VelRange
184 #else: self.dataOut.VelRange = None
184 #else: self.dataOut.VelRange = None
185
185
186 if hasattr(self.dataIn, 'RadarConst'): #Radar Constant
186 if hasattr(self.dataIn, 'RadarConst'): #Radar Constant
187 self.dataOut.RadarConst = self.dataIn.RadarConst
187 self.dataOut.RadarConst = self.dataIn.RadarConst
188
188
189 if hasattr(self.dataIn, 'NPW'): #NPW
189 if hasattr(self.dataIn, 'NPW'): #NPW
190 self.dataOut.NPW = self.dataIn.NPW
190 self.dataOut.NPW = self.dataIn.NPW
191
191
192 if hasattr(self.dataIn, 'COFA'): #COFA
192 if hasattr(self.dataIn, 'COFA'): #COFA
193 self.dataOut.COFA = self.dataIn.COFA
193 self.dataOut.COFA = self.dataIn.COFA
194
194
195
195
196
196
197 #---------------------- Correlation Data ---------------------------
197 #---------------------- Correlation Data ---------------------------
198
198
199 if self.dataIn.type == "Correlation":
199 if self.dataIn.type == "Correlation":
200 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.dataIn.splitFunctions()
200 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.dataIn.splitFunctions()
201
201
202 self.dataOut.data_pre = (self.dataIn.data_cf[acf_ind,:], self.dataIn.data_cf[ccf_ind,:,:])
202 self.dataOut.data_pre = (self.dataIn.data_cf[acf_ind,:], self.dataIn.data_cf[ccf_ind,:,:])
203 self.dataOut.normFactor = (self.dataIn.normFactor[acf_ind,:], self.dataIn.normFactor[ccf_ind,:])
203 self.dataOut.normFactor = (self.dataIn.normFactor[acf_ind,:], self.dataIn.normFactor[ccf_ind,:])
204 self.dataOut.groupList = (acf_pairs, ccf_pairs)
204 self.dataOut.groupList = (acf_pairs, ccf_pairs)
205
205
206 self.dataOut.abscissaList = self.dataIn.lagRange
206 self.dataOut.abscissaList = self.dataIn.lagRange
207 self.dataOut.noise = self.dataIn.noise
207 self.dataOut.noise = self.dataIn.noise
208 self.dataOut.data_snr = self.dataIn.SNR
208 self.dataOut.data_snr = self.dataIn.SNR
209 self.dataOut.flagNoData = False
209 self.dataOut.flagNoData = False
210 self.dataOut.nAvg = self.dataIn.nAvg
210 self.dataOut.nAvg = self.dataIn.nAvg
211
211
212 #---------------------- Parameters Data ---------------------------
212 #---------------------- Parameters Data ---------------------------
213
213
214 if self.dataIn.type == "Parameters":
214 if self.dataIn.type == "Parameters":
215 self.dataOut.copy(self.dataIn)
215 self.dataOut.copy(self.dataIn)
216 self.dataOut.flagNoData = False
216 self.dataOut.flagNoData = False
217 #print("yo si entre")
217 #print("yo si entre")
218
218
219 return True
219 return True
220
220
221 self.__updateObjFromInput()
221 self.__updateObjFromInput()
222 #print("yo si entre2")
222 #print("yo si entre2")
223
223
224 self.dataOut.utctimeInit = self.dataIn.utctime
224 self.dataOut.utctimeInit = self.dataIn.utctime
225 self.dataOut.paramInterval = self.dataIn.timeInterval
225 self.dataOut.paramInterval = self.dataIn.timeInterval
226 #print("soy spectra ",self.dataOut.utctimeInit)
226 #print("soy spectra ",self.dataOut.utctimeInit)
227 return
227 return
228
228
229
229
230 def target(tups):
230 def target(tups):
231
231
232 obj, args = tups
232 obj, args = tups
233
233
234 return obj.FitGau(args)
234 return obj.FitGau(args)
235
235
236 class RemoveWideGC(Operation):
236 class RemoveWideGC(Operation):
237 ''' This class remove the wide clutter and replace it with a simple interpolation points
237 ''' This class remove the wide clutter and replace it with a simple interpolation points
238 This mainly applies to CLAIRE radar
238 This mainly applies to CLAIRE radar
239
239
240 ClutterWidth : Width to look for the clutter peak
240 ClutterWidth : Width to look for the clutter peak
241
241
242 Input:
242 Input:
243
243
244 self.dataOut.data_pre : SPC and CSPC
244 self.dataOut.data_pre : SPC and CSPC
245 self.dataOut.spc_range : To select wind and rainfall velocities
245 self.dataOut.spc_range : To select wind and rainfall velocities
246
246
247 Affected:
247 Affected:
248
248
249 self.dataOut.data_pre : It is used for the new SPC and CSPC ranges of wind
249 self.dataOut.data_pre : It is used for the new SPC and CSPC ranges of wind
250
250
251 Written by D. Scipión 25.02.2021
251 Written by D. Scipión 25.02.2021
252 '''
252 '''
253 def __init__(self):
253 def __init__(self):
254 Operation.__init__(self)
254 Operation.__init__(self)
255 self.i = 0
255 self.i = 0
256 self.ich = 0
256 self.ich = 0
257 self.ir = 0
257 self.ir = 0
258
258
259 def run(self, dataOut, ClutterWidth=2.5):
259 def run(self, dataOut, ClutterWidth=2.5):
260 # print ('Entering RemoveWideGC ... ')
260 # print ('Entering RemoveWideGC ... ')
261
261
262 self.spc = dataOut.data_pre[0].copy()
262 self.spc = dataOut.data_pre[0].copy()
263 self.spc_out = dataOut.data_pre[0].copy()
263 self.spc_out = dataOut.data_pre[0].copy()
264 self.Num_Chn = self.spc.shape[0]
264 self.Num_Chn = self.spc.shape[0]
265 self.Num_Hei = self.spc.shape[2]
265 self.Num_Hei = self.spc.shape[2]
266 VelRange = dataOut.spc_range[2][:-1]
266 VelRange = dataOut.spc_range[2][:-1]
267 dv = VelRange[1]-VelRange[0]
267 dv = VelRange[1]-VelRange[0]
268
268
269 # Find the velocities that corresponds to zero
269 # Find the velocities that corresponds to zero
270 gc_values = numpy.squeeze(numpy.where(numpy.abs(VelRange) <= ClutterWidth))
270 gc_values = numpy.squeeze(numpy.where(numpy.abs(VelRange) <= ClutterWidth))
271
271
272 # Removing novalid data from the spectra
272 # Removing novalid data from the spectra
273 for ich in range(self.Num_Chn) :
273 for ich in range(self.Num_Chn) :
274 for ir in range(self.Num_Hei) :
274 for ir in range(self.Num_Hei) :
275 # Estimate the noise at each range
275 # Estimate the noise at each range
276 HSn = hildebrand_sekhon(self.spc[ich,:,ir],dataOut.nIncohInt)
276 HSn = hildebrand_sekhon(self.spc[ich,:,ir],dataOut.nIncohInt)
277
277
278 # Removing the noise floor at each range
278 # Removing the noise floor at each range
279 novalid = numpy.where(self.spc[ich,:,ir] < HSn)
279 novalid = numpy.where(self.spc[ich,:,ir] < HSn)
280 self.spc[ich,novalid,ir] = HSn
280 self.spc[ich,novalid,ir] = HSn
281
281
282 junk = numpy.append(numpy.insert(numpy.squeeze(self.spc[ich,gc_values,ir]),0,HSn),HSn)
282 junk = numpy.append(numpy.insert(numpy.squeeze(self.spc[ich,gc_values,ir]),0,HSn),HSn)
283 j1index = numpy.squeeze(numpy.where(numpy.diff(junk)>0))
283 j1index = numpy.squeeze(numpy.where(numpy.diff(junk)>0))
284 j2index = numpy.squeeze(numpy.where(numpy.diff(junk)<0))
284 j2index = numpy.squeeze(numpy.where(numpy.diff(junk)<0))
285 if ((numpy.size(j1index)<=1) | (numpy.size(j2index)<=1)) :
285 if ((numpy.size(j1index)<=1) | (numpy.size(j2index)<=1)) :
286 continue
286 continue
287 junk3 = numpy.squeeze(numpy.diff(j1index))
287 junk3 = numpy.squeeze(numpy.diff(j1index))
288 junk4 = numpy.squeeze(numpy.diff(j2index))
288 junk4 = numpy.squeeze(numpy.diff(j2index))
289
289
290 valleyindex = j2index[numpy.where(junk4>1)]
290 valleyindex = j2index[numpy.where(junk4>1)]
291 peakindex = j1index[numpy.where(junk3>1)]
291 peakindex = j1index[numpy.where(junk3>1)]
292
292
293 isvalid = numpy.squeeze(numpy.where(numpy.abs(VelRange[gc_values[peakindex]]) <= 2.5*dv))
293 isvalid = numpy.squeeze(numpy.where(numpy.abs(VelRange[gc_values[peakindex]]) <= 2.5*dv))
294 if numpy.size(isvalid) == 0 :
294 if numpy.size(isvalid) == 0 :
295 continue
295 continue
296 if numpy.size(isvalid) >1 :
296 if numpy.size(isvalid) >1 :
297 vindex = numpy.argmax(self.spc[ich,gc_values[peakindex[isvalid]],ir])
297 vindex = numpy.argmax(self.spc[ich,gc_values[peakindex[isvalid]],ir])
298 isvalid = isvalid[vindex]
298 isvalid = isvalid[vindex]
299
299
300 # clutter peak
300 # clutter peak
301 gcpeak = peakindex[isvalid]
301 gcpeak = peakindex[isvalid]
302 vl = numpy.where(valleyindex < gcpeak)
302 vl = numpy.where(valleyindex < gcpeak)
303 if numpy.size(vl) == 0:
303 if numpy.size(vl) == 0:
304 continue
304 continue
305 gcvl = valleyindex[vl[0][-1]]
305 gcvl = valleyindex[vl[0][-1]]
306 vr = numpy.where(valleyindex > gcpeak)
306 vr = numpy.where(valleyindex > gcpeak)
307 if numpy.size(vr) == 0:
307 if numpy.size(vr) == 0:
308 continue
308 continue
309 gcvr = valleyindex[vr[0][0]]
309 gcvr = valleyindex[vr[0][0]]
310
310
311 # Removing the clutter
311 # Removing the clutter
312 interpindex = numpy.array([gc_values[gcvl], gc_values[gcvr]])
312 interpindex = numpy.array([gc_values[gcvl], gc_values[gcvr]])
313 gcindex = gc_values[gcvl+1:gcvr-1]
313 gcindex = gc_values[gcvl+1:gcvr-1]
314 self.spc_out[ich,gcindex,ir] = numpy.interp(VelRange[gcindex],VelRange[interpindex],self.spc[ich,interpindex,ir])
314 self.spc_out[ich,gcindex,ir] = numpy.interp(VelRange[gcindex],VelRange[interpindex],self.spc[ich,interpindex,ir])
315
315
316 dataOut.data_pre[0] = self.spc_out
316 dataOut.data_pre[0] = self.spc_out
317 #print ('Leaving RemoveWideGC ... ')
317 #print ('Leaving RemoveWideGC ... ')
318 return dataOut
318 return dataOut
319
319
320 class SpectralFilters(Operation):
320 class SpectralFilters(Operation):
321 ''' This class allows to replace the novalid values with noise for each channel
321 ''' This class allows to replace the novalid values with noise for each channel
322 This applies to CLAIRE RADAR
322 This applies to CLAIRE RADAR
323
323
324 PositiveLimit : RightLimit of novalid data
324 PositiveLimit : RightLimit of novalid data
325 NegativeLimit : LeftLimit of novalid data
325 NegativeLimit : LeftLimit of novalid data
326
326
327 Input:
327 Input:
328
328
329 self.dataOut.data_pre : SPC and CSPC
329 self.dataOut.data_pre : SPC and CSPC
330 self.dataOut.spc_range : To select wind and rainfall velocities
330 self.dataOut.spc_range : To select wind and rainfall velocities
331
331
332 Affected:
332 Affected:
333
333
334 self.dataOut.data_pre : It is used for the new SPC and CSPC ranges of wind
334 self.dataOut.data_pre : It is used for the new SPC and CSPC ranges of wind
335
335
336 Written by D. Scipión 29.01.2021
336 Written by D. Scipión 29.01.2021
337 '''
337 '''
338 def __init__(self):
338 def __init__(self):
339 Operation.__init__(self)
339 Operation.__init__(self)
340 self.i = 0
340 self.i = 0
341
341
342 def run(self, dataOut, ):
342 def run(self, dataOut, ):
343
343
344 self.spc = dataOut.data_pre[0].copy()
344 self.spc = dataOut.data_pre[0].copy()
345 self.Num_Chn = self.spc.shape[0]
345 self.Num_Chn = self.spc.shape[0]
346 VelRange = dataOut.spc_range[2]
346 VelRange = dataOut.spc_range[2]
347
347
348 # novalid corresponds to data within the Negative and PositiveLimit
348 # novalid corresponds to data within the Negative and PositiveLimit
349
349
350
350
351 # Removing novalid data from the spectra
351 # Removing novalid data from the spectra
352 for i in range(self.Num_Chn):
352 for i in range(self.Num_Chn):
353 self.spc[i,novalid,:] = dataOut.noise[i]
353 self.spc[i,novalid,:] = dataOut.noise[i]
354 dataOut.data_pre[0] = self.spc
354 dataOut.data_pre[0] = self.spc
355 return dataOut
355 return dataOut
356
356
357 class GaussianFit(Operation):
357 class GaussianFit(Operation):
358
358
359 '''
359 '''
360 Function that fit of one and two generalized gaussians (gg) based
360 Function that fit of one and two generalized gaussians (gg) based
361 on the PSD shape across an "power band" identified from a cumsum of
361 on the PSD shape across an "power band" identified from a cumsum of
362 the measured spectrum - noise.
362 the measured spectrum - noise.
363
363
364 Input:
364 Input:
365 self.dataOut.data_pre : SelfSpectra
365 self.dataOut.data_pre : SelfSpectra
366
366
367 Output:
367 Output:
368 self.dataOut.SPCparam : SPC_ch1, SPC_ch2
368 self.dataOut.SPCparam : SPC_ch1, SPC_ch2
369
369
370 '''
370 '''
371 def __init__(self):
371 def __init__(self):
372 Operation.__init__(self)
372 Operation.__init__(self)
373 self.i=0
373 self.i=0
374
374
375
375
376 # def run(self, dataOut, num_intg=7, pnoise=1., SNRlimit=-9): #num_intg: Incoherent integrations, pnoise: Noise, vel_arr: range of velocities, similar to the ftt points
376 # def run(self, dataOut, num_intg=7, pnoise=1., SNRlimit=-9): #num_intg: Incoherent integrations, pnoise: Noise, vel_arr: range of velocities, similar to the ftt points
377 def run(self, dataOut, SNRdBlimit=-9, method='generalized'):
377 def run(self, dataOut, SNRdBlimit=-9, method='generalized'):
378 """This routine will find a couple of generalized Gaussians to a power spectrum
378 """This routine will find a couple of generalized Gaussians to a power spectrum
379 methods: generalized, squared
379 methods: generalized, squared
380 input: spc
380 input: spc
381 output:
381 output:
382 noise, amplitude0,shift0,width0,p0,Amplitude1,shift1,width1,p1
382 noise, amplitude0,shift0,width0,p0,Amplitude1,shift1,width1,p1
383 """
383 """
384 print ('Entering ',method,' double Gaussian fit')
384 print ('Entering ',method,' double Gaussian fit')
385 self.spc = dataOut.data_pre[0].copy()
385 self.spc = dataOut.data_pre[0].copy()
386 self.Num_Hei = self.spc.shape[2]
386 self.Num_Hei = self.spc.shape[2]
387 self.Num_Bin = self.spc.shape[1]
387 self.Num_Bin = self.spc.shape[1]
388 self.Num_Chn = self.spc.shape[0]
388 self.Num_Chn = self.spc.shape[0]
389
389
390 start_time = time.time()
390 start_time = time.time()
391
391
392 pool = Pool(processes=self.Num_Chn)
392 pool = Pool(processes=self.Num_Chn)
393 args = [(dataOut.spc_range[2], ich, dataOut.spc_noise[ich], dataOut.nIncohInt, SNRdBlimit) for ich in range(self.Num_Chn)]
393 args = [(dataOut.spc_range[2], ich, dataOut.spc_noise[ich], dataOut.nIncohInt, SNRdBlimit) for ich in range(self.Num_Chn)]
394 objs = [self for __ in range(self.Num_Chn)]
394 objs = [self for __ in range(self.Num_Chn)]
395 attrs = list(zip(objs, args))
395 attrs = list(zip(objs, args))
396 DGauFitParam = pool.map(target, attrs)
396 DGauFitParam = pool.map(target, attrs)
397 # Parameters:
397 # Parameters:
398 # 0. Noise, 1. Amplitude, 2. Shift, 3. Width 4. Power
398 # 0. Noise, 1. Amplitude, 2. Shift, 3. Width 4. Power
399 dataOut.DGauFitParams = numpy.asarray(DGauFitParam)
399 dataOut.DGauFitParams = numpy.asarray(DGauFitParam)
400
400
401 # Double Gaussian Curves
401 # Double Gaussian Curves
402 gau0 = numpy.zeros([self.Num_Chn,self.Num_Bin,self.Num_Hei])
402 gau0 = numpy.zeros([self.Num_Chn,self.Num_Bin,self.Num_Hei])
403 gau0[:] = numpy.NaN
403 gau0[:] = numpy.NaN
404 gau1 = numpy.zeros([self.Num_Chn,self.Num_Bin,self.Num_Hei])
404 gau1 = numpy.zeros([self.Num_Chn,self.Num_Bin,self.Num_Hei])
405 gau1[:] = numpy.NaN
405 gau1[:] = numpy.NaN
406 x_mtr = numpy.transpose(numpy.tile(dataOut.getVelRange(1)[:-1], (self.Num_Hei,1)))
406 x_mtr = numpy.transpose(numpy.tile(dataOut.getVelRange(1)[:-1], (self.Num_Hei,1)))
407 for iCh in range(self.Num_Chn):
407 for iCh in range(self.Num_Chn):
408 N0 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][0,:,0]] * self.Num_Bin))
408 N0 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][0,:,0]] * self.Num_Bin))
409 N1 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][0,:,1]] * self.Num_Bin))
409 N1 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][0,:,1]] * self.Num_Bin))
410 A0 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][1,:,0]] * self.Num_Bin))
410 A0 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][1,:,0]] * self.Num_Bin))
411 A1 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][1,:,1]] * self.Num_Bin))
411 A1 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][1,:,1]] * self.Num_Bin))
412 v0 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][2,:,0]] * self.Num_Bin))
412 v0 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][2,:,0]] * self.Num_Bin))
413 v1 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][2,:,1]] * self.Num_Bin))
413 v1 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][2,:,1]] * self.Num_Bin))
414 s0 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][3,:,0]] * self.Num_Bin))
414 s0 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][3,:,0]] * self.Num_Bin))
415 s1 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][3,:,1]] * self.Num_Bin))
415 s1 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][3,:,1]] * self.Num_Bin))
416 if method == 'genealized':
416 if method == 'genealized':
417 p0 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][4,:,0]] * self.Num_Bin))
417 p0 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][4,:,0]] * self.Num_Bin))
418 p1 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][4,:,1]] * self.Num_Bin))
418 p1 = numpy.transpose(numpy.transpose([dataOut.DGauFitParams[iCh][4,:,1]] * self.Num_Bin))
419 elif method == 'squared':
419 elif method == 'squared':
420 p0 = 2.
420 p0 = 2.
421 p1 = 2.
421 p1 = 2.
422 gau0[iCh] = A0*numpy.exp(-0.5*numpy.abs((x_mtr-v0)/s0)**p0)+N0
422 gau0[iCh] = A0*numpy.exp(-0.5*numpy.abs((x_mtr-v0)/s0)**p0)+N0
423 gau1[iCh] = A1*numpy.exp(-0.5*numpy.abs((x_mtr-v1)/s1)**p1)+N1
423 gau1[iCh] = A1*numpy.exp(-0.5*numpy.abs((x_mtr-v1)/s1)**p1)+N1
424 dataOut.GaussFit0 = gau0
424 dataOut.GaussFit0 = gau0
425 dataOut.GaussFit1 = gau1
425 dataOut.GaussFit1 = gau1
426
426
427 print('Leaving ',method ,' double Gaussian fit')
427 print('Leaving ',method ,' double Gaussian fit')
428 return dataOut
428 return dataOut
429
429
430 def FitGau(self, X):
430 def FitGau(self, X):
431 # print('Entering FitGau')
431 # print('Entering FitGau')
432 # Assigning the variables
432 # Assigning the variables
433 Vrange, ch, wnoise, num_intg, SNRlimit = X
433 Vrange, ch, wnoise, num_intg, SNRlimit = X
434 # Noise Limits
434 # Noise Limits
435 noisebl = wnoise * 0.9
435 noisebl = wnoise * 0.9
436 noisebh = wnoise * 1.1
436 noisebh = wnoise * 1.1
437 # Radar Velocity
437 # Radar Velocity
438 Va = max(Vrange)
438 Va = max(Vrange)
439 deltav = Vrange[1] - Vrange[0]
439 deltav = Vrange[1] - Vrange[0]
440 x = numpy.arange(self.Num_Bin)
440 x = numpy.arange(self.Num_Bin)
441
441
442 # print ('stop 0')
442 # print ('stop 0')
443
443
444 # 5 parameters, 2 Gaussians
444 # 5 parameters, 2 Gaussians
445 DGauFitParam = numpy.zeros([5, self.Num_Hei,2])
445 DGauFitParam = numpy.zeros([5, self.Num_Hei,2])
446 DGauFitParam[:] = numpy.NaN
446 DGauFitParam[:] = numpy.NaN
447
447
448 # SPCparam = []
448 # SPCparam = []
449 # SPC_ch1 = numpy.zeros([self.Num_Bin,self.Num_Hei])
449 # SPC_ch1 = numpy.zeros([self.Num_Bin,self.Num_Hei])
450 # SPC_ch2 = numpy.zeros([self.Num_Bin,self.Num_Hei])
450 # SPC_ch2 = numpy.zeros([self.Num_Bin,self.Num_Hei])
451 # SPC_ch1[:] = 0 #numpy.NaN
451 # SPC_ch1[:] = 0 #numpy.NaN
452 # SPC_ch2[:] = 0 #numpy.NaN
452 # SPC_ch2[:] = 0 #numpy.NaN
453 # print ('stop 1')
453 # print ('stop 1')
454 for ht in range(self.Num_Hei):
454 for ht in range(self.Num_Hei):
455 # print (ht)
455 # print (ht)
456 # print ('stop 2')
456 # print ('stop 2')
457 # Spectra at each range
457 # Spectra at each range
458 spc = numpy.asarray(self.spc)[ch,:,ht]
458 spc = numpy.asarray(self.spc)[ch,:,ht]
459 snr = ( spc.mean() - wnoise ) / wnoise
459 snr = ( spc.mean() - wnoise ) / wnoise
460 snrdB = 10.*numpy.log10(snr)
460 snrdB = 10.*numpy.log10(snr)
461
461
462 #print ('stop 3')
462 #print ('stop 3')
463 if snrdB < SNRlimit :
463 if snrdB < SNRlimit :
464 # snr = numpy.NaN
464 # snr = numpy.NaN
465 # SPC_ch1[:,ht] = 0#numpy.NaN
465 # SPC_ch1[:,ht] = 0#numpy.NaN
466 # SPC_ch1[:,ht] = 0#numpy.NaN
466 # SPC_ch1[:,ht] = 0#numpy.NaN
467 # SPCparam = (SPC_ch1,SPC_ch2)
467 # SPCparam = (SPC_ch1,SPC_ch2)
468 # print ('SNR less than SNRth')
468 # print ('SNR less than SNRth')
469 continue
469 continue
470 # wnoise = hildebrand_sekhon(spc,num_intg)
470 # wnoise = hildebrand_sekhon(spc,num_intg)
471 # print ('stop 2.01')
471 # print ('stop 2.01')
472 #############################################
472 #############################################
473 # normalizing spc and noise
473 # normalizing spc and noise
474 # This part differs from gg1
474 # This part differs from gg1
475 # spc_norm_max = max(spc) #commented by D. Scipión 19.03.2021
475 # spc_norm_max = max(spc) #commented by D. Scipión 19.03.2021
476 #spc = spc / spc_norm_max
476 #spc = spc / spc_norm_max
477 # pnoise = pnoise #/ spc_norm_max #commented by D. Scipión 19.03.2021
477 # pnoise = pnoise #/ spc_norm_max #commented by D. Scipión 19.03.2021
478 #############################################
478 #############################################
479
479
480 # print ('stop 2.1')
480 # print ('stop 2.1')
481 fatspectra=1.0
481 fatspectra=1.0
482 # noise per channel.... we might want to use the noise at each range
482 # noise per channel.... we might want to use the noise at each range
483
483
484 # wnoise = noise_ #/ spc_norm_max #commented by D. Scipión 19.03.2021
484 # wnoise = noise_ #/ spc_norm_max #commented by D. Scipión 19.03.2021
485 #wnoise,stdv,i_max,index =enoise(spc,num_intg) #noise estimate using Hildebrand Sekhon, only wnoise is used
485 #wnoise,stdv,i_max,index =enoise(spc,num_intg) #noise estimate using Hildebrand Sekhon, only wnoise is used
486 #if wnoise>1.1*pnoise: # to be tested later
486 #if wnoise>1.1*pnoise: # to be tested later
487 # wnoise=pnoise
487 # wnoise=pnoise
488 # noisebl = wnoise*0.9
488 # noisebl = wnoise*0.9
489 # noisebh = wnoise*1.1
489 # noisebh = wnoise*1.1
490 spc = spc - wnoise # signal
490 spc = spc - wnoise # signal
491
491
492 # print ('stop 2.2')
492 # print ('stop 2.2')
493 minx = numpy.argmin(spc)
493 minx = numpy.argmin(spc)
494 #spcs=spc.copy()
494 #spcs=spc.copy()
495 spcs = numpy.roll(spc,-minx)
495 spcs = numpy.roll(spc,-minx)
496 cum = numpy.cumsum(spcs)
496 cum = numpy.cumsum(spcs)
497 # tot_noise = wnoise * self.Num_Bin #64;
497 # tot_noise = wnoise * self.Num_Bin #64;
498
498
499 # print ('stop 2.3')
499 # print ('stop 2.3')
500 # snr = sum(spcs) / tot_noise
500 # snr = sum(spcs) / tot_noise
501 # snrdB = 10.*numpy.log10(snr)
501 # snrdB = 10.*numpy.log10(snr)
502 #print ('stop 3')
502 #print ('stop 3')
503 # if snrdB < SNRlimit :
503 # if snrdB < SNRlimit :
504 # snr = numpy.NaN
504 # snr = numpy.NaN
505 # SPC_ch1[:,ht] = 0#numpy.NaN
505 # SPC_ch1[:,ht] = 0#numpy.NaN
506 # SPC_ch1[:,ht] = 0#numpy.NaN
506 # SPC_ch1[:,ht] = 0#numpy.NaN
507 # SPCparam = (SPC_ch1,SPC_ch2)
507 # SPCparam = (SPC_ch1,SPC_ch2)
508 # print ('SNR less than SNRth')
508 # print ('SNR less than SNRth')
509 # continue
509 # continue
510
510
511
511
512 #if snrdB<-18 or numpy.isnan(snrdB) or num_intg<4:
512 #if snrdB<-18 or numpy.isnan(snrdB) or num_intg<4:
513 # return [None,]*4,[None,]*4,None,snrdB,None,None,[None,]*5,[None,]*9,None
513 # return [None,]*4,[None,]*4,None,snrdB,None,None,[None,]*5,[None,]*9,None
514 # print ('stop 4')
514 # print ('stop 4')
515 cummax = max(cum)
515 cummax = max(cum)
516 epsi = 0.08 * fatspectra # cumsum to narrow down the energy region
516 epsi = 0.08 * fatspectra # cumsum to narrow down the energy region
517 cumlo = cummax * epsi
517 cumlo = cummax * epsi
518 cumhi = cummax * (1-epsi)
518 cumhi = cummax * (1-epsi)
519 powerindex = numpy.array(numpy.where(numpy.logical_and(cum>cumlo, cum<cumhi))[0])
519 powerindex = numpy.array(numpy.where(numpy.logical_and(cum>cumlo, cum<cumhi))[0])
520
520
521 # print ('stop 5')
521 # print ('stop 5')
522 if len(powerindex) < 1:# case for powerindex 0
522 if len(powerindex) < 1:# case for powerindex 0
523 # print ('powerindex < 1')
523 # print ('powerindex < 1')
524 continue
524 continue
525 powerlo = powerindex[0]
525 powerlo = powerindex[0]
526 powerhi = powerindex[-1]
526 powerhi = powerindex[-1]
527 powerwidth = powerhi-powerlo
527 powerwidth = powerhi-powerlo
528 if powerwidth <= 1:
528 if powerwidth <= 1:
529 # print('powerwidth <= 1')
529 # print('powerwidth <= 1')
530 continue
530 continue
531
531
532 # print ('stop 6')
532 # print ('stop 6')
533 firstpeak = powerlo + powerwidth/10.# first gaussian energy location
533 firstpeak = powerlo + powerwidth/10.# first gaussian energy location
534 secondpeak = powerhi - powerwidth/10. #second gaussian energy location
534 secondpeak = powerhi - powerwidth/10. #second gaussian energy location
535 midpeak = (firstpeak + secondpeak)/2.
535 midpeak = (firstpeak + secondpeak)/2.
536 firstamp = spcs[int(firstpeak)]
536 firstamp = spcs[int(firstpeak)]
537 secondamp = spcs[int(secondpeak)]
537 secondamp = spcs[int(secondpeak)]
538 midamp = spcs[int(midpeak)]
538 midamp = spcs[int(midpeak)]
539
539
540 y_data = spc + wnoise
540 y_data = spc + wnoise
541
541
542 ''' single Gaussian '''
542 ''' single Gaussian '''
543 shift0 = numpy.mod(midpeak+minx, self.Num_Bin )
543 shift0 = numpy.mod(midpeak+minx, self.Num_Bin )
544 width0 = powerwidth/4.#Initialization entire power of spectrum divided by 4
544 width0 = powerwidth/4.#Initialization entire power of spectrum divided by 4
545 power0 = 2.
545 power0 = 2.
546 amplitude0 = midamp
546 amplitude0 = midamp
547 state0 = [shift0,width0,amplitude0,power0,wnoise]
547 state0 = [shift0,width0,amplitude0,power0,wnoise]
548 bnds = ((0,self.Num_Bin-1),(1,powerwidth),(0,None),(0.5,3.),(noisebl,noisebh))
548 bnds = ((0,self.Num_Bin-1),(1,powerwidth),(0,None),(0.5,3.),(noisebl,noisebh))
549 lsq1 = fmin_l_bfgs_b(self.misfit1, state0, args=(y_data,x,num_intg), bounds=bnds, approx_grad=True)
549 lsq1 = fmin_l_bfgs_b(self.misfit1, state0, args=(y_data,x,num_intg), bounds=bnds, approx_grad=True)
550 # print ('stop 7.1')
550 # print ('stop 7.1')
551 # print (bnds)
551 # print (bnds)
552
552
553 chiSq1=lsq1[1]
553 chiSq1=lsq1[1]
554
554
555 # print ('stop 8')
555 # print ('stop 8')
556 if fatspectra<1.0 and powerwidth<4:
556 if fatspectra<1.0 and powerwidth<4:
557 choice=0
557 choice=0
558 Amplitude0=lsq1[0][2]
558 Amplitude0=lsq1[0][2]
559 shift0=lsq1[0][0]
559 shift0=lsq1[0][0]
560 width0=lsq1[0][1]
560 width0=lsq1[0][1]
561 p0=lsq1[0][3]
561 p0=lsq1[0][3]
562 Amplitude1=0.
562 Amplitude1=0.
563 shift1=0.
563 shift1=0.
564 width1=0.
564 width1=0.
565 p1=0.
565 p1=0.
566 noise=lsq1[0][4]
566 noise=lsq1[0][4]
567 #return (numpy.array([shift0,width0,Amplitude0,p0]),
567 #return (numpy.array([shift0,width0,Amplitude0,p0]),
568 # numpy.array([shift1,width1,Amplitude1,p1]),noise,snrdB,chiSq1,6.,sigmas1,[None,]*9,choice)
568 # numpy.array([shift1,width1,Amplitude1,p1]),noise,snrdB,chiSq1,6.,sigmas1,[None,]*9,choice)
569
569
570 # print ('stop 9')
570 # print ('stop 9')
571 ''' two Gaussians '''
571 ''' two Gaussians '''
572 #shift0=numpy.mod(firstpeak+minx,64); shift1=numpy.mod(secondpeak+minx,64)
572 #shift0=numpy.mod(firstpeak+minx,64); shift1=numpy.mod(secondpeak+minx,64)
573 shift0 = numpy.mod(firstpeak+minx, self.Num_Bin )
573 shift0 = numpy.mod(firstpeak+minx, self.Num_Bin )
574 shift1 = numpy.mod(secondpeak+minx, self.Num_Bin )
574 shift1 = numpy.mod(secondpeak+minx, self.Num_Bin )
575 width0 = powerwidth/6.
575 width0 = powerwidth/6.
576 width1 = width0
576 width1 = width0
577 power0 = 2.
577 power0 = 2.
578 power1 = power0
578 power1 = power0
579 amplitude0 = firstamp
579 amplitude0 = firstamp
580 amplitude1 = secondamp
580 amplitude1 = secondamp
581 state0 = [shift0,width0,amplitude0,power0,shift1,width1,amplitude1,power1,wnoise]
581 state0 = [shift0,width0,amplitude0,power0,shift1,width1,amplitude1,power1,wnoise]
582 #bnds=((0,63),(1,powerwidth/2.),(0,None),(0.5,3.),(0,63),(1,powerwidth/2.),(0,None),(0.5,3.),(noisebl,noisebh))
582 #bnds=((0,63),(1,powerwidth/2.),(0,None),(0.5,3.),(0,63),(1,powerwidth/2.),(0,None),(0.5,3.),(noisebl,noisebh))
583 bnds=((0,self.Num_Bin-1),(1,powerwidth/2.),(0,None),(0.5,3.),(0,self.Num_Bin-1),(1,powerwidth/2.),(0,None),(0.5,3.),(noisebl,noisebh))
583 bnds=((0,self.Num_Bin-1),(1,powerwidth/2.),(0,None),(0.5,3.),(0,self.Num_Bin-1),(1,powerwidth/2.),(0,None),(0.5,3.),(noisebl,noisebh))
584 #bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth/2.),(0,None),(0.5,3.),( 0,(self.Num_Bin-1)),(1,powerwidth/2.),(0,None),(0.5,3.),(0.1,0.5))
584 #bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth/2.),(0,None),(0.5,3.),( 0,(self.Num_Bin-1)),(1,powerwidth/2.),(0,None),(0.5,3.),(0.1,0.5))
585
585
586 # print ('stop 10')
586 # print ('stop 10')
587 lsq2 = fmin_l_bfgs_b( self.misfit2 , state0 , args=(y_data,x,num_intg) , bounds=bnds , approx_grad=True )
587 lsq2 = fmin_l_bfgs_b( self.misfit2 , state0 , args=(y_data,x,num_intg) , bounds=bnds , approx_grad=True )
588
588
589 # print ('stop 11')
589 # print ('stop 11')
590 chiSq2 = lsq2[1]
590 chiSq2 = lsq2[1]
591
591
592 # print ('stop 12')
592 # print ('stop 12')
593
593
594 oneG = (chiSq1<5 and chiSq1/chiSq2<2.0) and (abs(lsq2[0][0]-lsq2[0][4])<(lsq2[0][1]+lsq2[0][5])/3. or abs(lsq2[0][0]-lsq2[0][4])<10)
594 oneG = (chiSq1<5 and chiSq1/chiSq2<2.0) and (abs(lsq2[0][0]-lsq2[0][4])<(lsq2[0][1]+lsq2[0][5])/3. or abs(lsq2[0][0]-lsq2[0][4])<10)
595
595
596 # print ('stop 13')
596 # print ('stop 13')
597 if snrdB>-12: # when SNR is strong pick the peak with least shift (LOS velocity) error
597 if snrdB>-12: # when SNR is strong pick the peak with least shift (LOS velocity) error
598 if oneG:
598 if oneG:
599 choice = 0
599 choice = 0
600 else:
600 else:
601 w1 = lsq2[0][1]; w2 = lsq2[0][5]
601 w1 = lsq2[0][1]; w2 = lsq2[0][5]
602 a1 = lsq2[0][2]; a2 = lsq2[0][6]
602 a1 = lsq2[0][2]; a2 = lsq2[0][6]
603 p1 = lsq2[0][3]; p2 = lsq2[0][7]
603 p1 = lsq2[0][3]; p2 = lsq2[0][7]
604 s1 = (2**(1+1./p1))*scipy.special.gamma(1./p1)/p1
604 s1 = (2**(1+1./p1))*scipy.special.gamma(1./p1)/p1
605 s2 = (2**(1+1./p2))*scipy.special.gamma(1./p2)/p2
605 s2 = (2**(1+1./p2))*scipy.special.gamma(1./p2)/p2
606 gp1 = a1*w1*s1; gp2 = a2*w2*s2 # power content of each ggaussian with proper p scaling
606 gp1 = a1*w1*s1; gp2 = a2*w2*s2 # power content of each ggaussian with proper p scaling
607
607
608 if gp1>gp2:
608 if gp1>gp2:
609 if a1>0.7*a2:
609 if a1>0.7*a2:
610 choice = 1
610 choice = 1
611 else:
611 else:
612 choice = 2
612 choice = 2
613 elif gp2>gp1:
613 elif gp2>gp1:
614 if a2>0.7*a1:
614 if a2>0.7*a1:
615 choice = 2
615 choice = 2
616 else:
616 else:
617 choice = 1
617 choice = 1
618 else:
618 else:
619 choice = numpy.argmax([a1,a2])+1
619 choice = numpy.argmax([a1,a2])+1
620 #else:
620 #else:
621 #choice=argmin([std2a,std2b])+1
621 #choice=argmin([std2a,std2b])+1
622
622
623 else: # with low SNR go to the most energetic peak
623 else: # with low SNR go to the most energetic peak
624 choice = numpy.argmax([lsq1[0][2]*lsq1[0][1],lsq2[0][2]*lsq2[0][1],lsq2[0][6]*lsq2[0][5]])
624 choice = numpy.argmax([lsq1[0][2]*lsq1[0][1],lsq2[0][2]*lsq2[0][1],lsq2[0][6]*lsq2[0][5]])
625
625
626 # print ('stop 14')
626 # print ('stop 14')
627 shift0 = lsq2[0][0]
627 shift0 = lsq2[0][0]
628 vel0 = Vrange[0] + shift0 * deltav
628 vel0 = Vrange[0] + shift0 * deltav
629 shift1 = lsq2[0][4]
629 shift1 = lsq2[0][4]
630 # vel1=Vrange[0] + shift1 * deltav
630 # vel1=Vrange[0] + shift1 * deltav
631
631
632 # max_vel = 1.0
632 # max_vel = 1.0
633 # Va = max(Vrange)
633 # Va = max(Vrange)
634 # deltav = Vrange[1]-Vrange[0]
634 # deltav = Vrange[1]-Vrange[0]
635 # print ('stop 15')
635 # print ('stop 15')
636 #first peak will be 0, second peak will be 1
636 #first peak will be 0, second peak will be 1
637 # if vel0 > -1.0 and vel0 < max_vel : #first peak is in the correct range # Commented by D.Scipión 19.03.2021
637 # if vel0 > -1.0 and vel0 < max_vel : #first peak is in the correct range # Commented by D.Scipión 19.03.2021
638 if vel0 > -Va and vel0 < Va : #first peak is in the correct range
638 if vel0 > -Va and vel0 < Va : #first peak is in the correct range
639 shift0 = lsq2[0][0]
639 shift0 = lsq2[0][0]
640 width0 = lsq2[0][1]
640 width0 = lsq2[0][1]
641 Amplitude0 = lsq2[0][2]
641 Amplitude0 = lsq2[0][2]
642 p0 = lsq2[0][3]
642 p0 = lsq2[0][3]
643
643
644 shift1 = lsq2[0][4]
644 shift1 = lsq2[0][4]
645 width1 = lsq2[0][5]
645 width1 = lsq2[0][5]
646 Amplitude1 = lsq2[0][6]
646 Amplitude1 = lsq2[0][6]
647 p1 = lsq2[0][7]
647 p1 = lsq2[0][7]
648 noise = lsq2[0][8]
648 noise = lsq2[0][8]
649 else:
649 else:
650 shift1 = lsq2[0][0]
650 shift1 = lsq2[0][0]
651 width1 = lsq2[0][1]
651 width1 = lsq2[0][1]
652 Amplitude1 = lsq2[0][2]
652 Amplitude1 = lsq2[0][2]
653 p1 = lsq2[0][3]
653 p1 = lsq2[0][3]
654
654
655 shift0 = lsq2[0][4]
655 shift0 = lsq2[0][4]
656 width0 = lsq2[0][5]
656 width0 = lsq2[0][5]
657 Amplitude0 = lsq2[0][6]
657 Amplitude0 = lsq2[0][6]
658 p0 = lsq2[0][7]
658 p0 = lsq2[0][7]
659 noise = lsq2[0][8]
659 noise = lsq2[0][8]
660
660
661 if Amplitude0<0.05: # in case the peak is noise
661 if Amplitude0<0.05: # in case the peak is noise
662 shift0,width0,Amplitude0,p0 = 4*[numpy.NaN]
662 shift0,width0,Amplitude0,p0 = 4*[numpy.NaN]
663 if Amplitude1<0.05:
663 if Amplitude1<0.05:
664 shift1,width1,Amplitude1,p1 = 4*[numpy.NaN]
664 shift1,width1,Amplitude1,p1 = 4*[numpy.NaN]
665
665
666 # print ('stop 16 ')
666 # print ('stop 16 ')
667 # SPC_ch1[:,ht] = noise + Amplitude0*numpy.exp(-0.5*(abs(x-shift0)/width0)**p0)
667 # SPC_ch1[:,ht] = noise + Amplitude0*numpy.exp(-0.5*(abs(x-shift0)/width0)**p0)
668 # SPC_ch2[:,ht] = noise + Amplitude1*numpy.exp(-0.5*(abs(x-shift1)/width1)**p1)
668 # SPC_ch2[:,ht] = noise + Amplitude1*numpy.exp(-0.5*(abs(x-shift1)/width1)**p1)
669 # SPCparam = (SPC_ch1,SPC_ch2)
669 # SPCparam = (SPC_ch1,SPC_ch2)
670
670
671 DGauFitParam[0,ht,0] = noise
671 DGauFitParam[0,ht,0] = noise
672 DGauFitParam[0,ht,1] = noise
672 DGauFitParam[0,ht,1] = noise
673 DGauFitParam[1,ht,0] = Amplitude0
673 DGauFitParam[1,ht,0] = Amplitude0
674 DGauFitParam[1,ht,1] = Amplitude1
674 DGauFitParam[1,ht,1] = Amplitude1
675 DGauFitParam[2,ht,0] = Vrange[0] + shift0 * deltav
675 DGauFitParam[2,ht,0] = Vrange[0] + shift0 * deltav
676 DGauFitParam[2,ht,1] = Vrange[0] + shift1 * deltav
676 DGauFitParam[2,ht,1] = Vrange[0] + shift1 * deltav
677 DGauFitParam[3,ht,0] = width0 * deltav
677 DGauFitParam[3,ht,0] = width0 * deltav
678 DGauFitParam[3,ht,1] = width1 * deltav
678 DGauFitParam[3,ht,1] = width1 * deltav
679 DGauFitParam[4,ht,0] = p0
679 DGauFitParam[4,ht,0] = p0
680 DGauFitParam[4,ht,1] = p1
680 DGauFitParam[4,ht,1] = p1
681
681
682 # print (DGauFitParam.shape)
682 # print (DGauFitParam.shape)
683 # print ('Leaving FitGau')
683 # print ('Leaving FitGau')
684 return DGauFitParam
684 return DGauFitParam
685 # return SPCparam
685 # return SPCparam
686 # return GauSPC
686 # return GauSPC
687
687
688 def y_model1(self,x,state):
688 def y_model1(self,x,state):
689 shift0, width0, amplitude0, power0, noise = state
689 shift0, width0, amplitude0, power0, noise = state
690 model0 = amplitude0*numpy.exp(-0.5*abs((x - shift0)/width0)**power0)
690 model0 = amplitude0*numpy.exp(-0.5*abs((x - shift0)/width0)**power0)
691 model0u = amplitude0*numpy.exp(-0.5*abs((x - shift0 - self.Num_Bin)/width0)**power0)
691 model0u = amplitude0*numpy.exp(-0.5*abs((x - shift0 - self.Num_Bin)/width0)**power0)
692 model0d = amplitude0*numpy.exp(-0.5*abs((x - shift0 + self.Num_Bin)/width0)**power0)
692 model0d = amplitude0*numpy.exp(-0.5*abs((x - shift0 + self.Num_Bin)/width0)**power0)
693 return model0 + model0u + model0d + noise
693 return model0 + model0u + model0d + noise
694
694
695 def y_model2(self,x,state): #Equation for two generalized Gaussians with Nyquist
695 def y_model2(self,x,state): #Equation for two generalized Gaussians with Nyquist
696 shift0, width0, amplitude0, power0, shift1, width1, amplitude1, power1, noise = state
696 shift0, width0, amplitude0, power0, shift1, width1, amplitude1, power1, noise = state
697 model0 = amplitude0*numpy.exp(-0.5*abs((x-shift0)/width0)**power0)
697 model0 = amplitude0*numpy.exp(-0.5*abs((x-shift0)/width0)**power0)
698 model0u = amplitude0*numpy.exp(-0.5*abs((x - shift0 - self.Num_Bin)/width0)**power0)
698 model0u = amplitude0*numpy.exp(-0.5*abs((x - shift0 - self.Num_Bin)/width0)**power0)
699 model0d = amplitude0*numpy.exp(-0.5*abs((x - shift0 + self.Num_Bin)/width0)**power0)
699 model0d = amplitude0*numpy.exp(-0.5*abs((x - shift0 + self.Num_Bin)/width0)**power0)
700
700
701 model1 = amplitude1*numpy.exp(-0.5*abs((x - shift1)/width1)**power1)
701 model1 = amplitude1*numpy.exp(-0.5*abs((x - shift1)/width1)**power1)
702 model1u = amplitude1*numpy.exp(-0.5*abs((x - shift1 - self.Num_Bin)/width1)**power1)
702 model1u = amplitude1*numpy.exp(-0.5*abs((x - shift1 - self.Num_Bin)/width1)**power1)
703 model1d = amplitude1*numpy.exp(-0.5*abs((x - shift1 + self.Num_Bin)/width1)**power1)
703 model1d = amplitude1*numpy.exp(-0.5*abs((x - shift1 + self.Num_Bin)/width1)**power1)
704 return model0 + model0u + model0d + model1 + model1u + model1d + noise
704 return model0 + model0u + model0d + model1 + model1u + model1d + noise
705
705
706 def misfit1(self,state,y_data,x,num_intg): # This function compares how close real data is with the model data, the close it is, the better it is.
706 def misfit1(self,state,y_data,x,num_intg): # This function compares how close real data is with the model data, the close it is, the better it is.
707
707
708 return num_intg*sum((numpy.log(y_data)-numpy.log(self.y_model1(x,state)))**2)#/(64-5.) # /(64-5.) can be commented
708 return num_intg*sum((numpy.log(y_data)-numpy.log(self.y_model1(x,state)))**2)#/(64-5.) # /(64-5.) can be commented
709
709
710 def misfit2(self,state,y_data,x,num_intg):
710 def misfit2(self,state,y_data,x,num_intg):
711 return num_intg*sum((numpy.log(y_data)-numpy.log(self.y_model2(x,state)))**2)#/(64-9.)
711 return num_intg*sum((numpy.log(y_data)-numpy.log(self.y_model2(x,state)))**2)#/(64-9.)
712
712
713
713
714
714
715 class PrecipitationProc(Operation):
715 class PrecipitationProc(Operation):
716
716
717 '''
717 '''
718 Operator that estimates Reflectivity factor (Z), and estimates rainfall Rate (R)
718 Operator that estimates Reflectivity factor (Z), and estimates rainfall Rate (R)
719
719
720 Input:
720 Input:
721 self.dataOut.data_pre : SelfSpectra
721 self.dataOut.data_pre : SelfSpectra
722
722
723 Output:
723 Output:
724
724
725 self.dataOut.data_output : Reflectivity factor, rainfall Rate
725 self.dataOut.data_output : Reflectivity factor, rainfall Rate
726
726
727
727
728 Parameters affected:
728 Parameters affected:
729 '''
729 '''
730
730
731 def __init__(self):
731 def __init__(self):
732 Operation.__init__(self)
732 Operation.__init__(self)
733 self.i=0
733 self.i=0
734
734
735 def run(self, dataOut, radar=None, Pt=5000, Gt=295.1209, Gr=70.7945, Lambda=0.6741, aL=2.5118,
735 def run(self, dataOut, radar=None, Pt=5000, Gt=295.1209, Gr=70.7945, Lambda=0.6741, aL=2.5118,
736 tauW=4e-06, ThetaT=0.1656317, ThetaR=0.36774087, Km2 = 0.93, Altitude=3350,SNRdBlimit=-30):
736 tauW=4e-06, ThetaT=0.1656317, ThetaR=0.36774087, Km2 = 0.93, Altitude=3350,SNRdBlimit=-30):
737
737
738 # print ('Entering PrecepitationProc ... ')
738 # print ('Entering PrecepitationProc ... ')
739
739
740 if radar == "MIRA35C" :
740 if radar == "MIRA35C" :
741
741
742 self.spc = dataOut.data_pre[0].copy()
742 self.spc = dataOut.data_pre[0].copy()
743 self.Num_Hei = self.spc.shape[2]
743 self.Num_Hei = self.spc.shape[2]
744 self.Num_Bin = self.spc.shape[1]
744 self.Num_Bin = self.spc.shape[1]
745 self.Num_Chn = self.spc.shape[0]
745 self.Num_Chn = self.spc.shape[0]
746 Ze = self.dBZeMODE2(dataOut)
746 Ze = self.dBZeMODE2(dataOut)
747
747
748 else:
748 else:
749
749
750 self.spc = dataOut.data_pre[0].copy()
750 self.spc = dataOut.data_pre[0].copy()
751
751
752 #NOTA SE DEBE REMOVER EL RANGO DEL PULSO TX
752 #NOTA SE DEBE REMOVER EL RANGO DEL PULSO TX
753 self.spc[:,:,0:7]= numpy.NaN
753 self.spc[:,:,0:7]= numpy.NaN
754
754
755 self.Num_Hei = self.spc.shape[2]
755 self.Num_Hei = self.spc.shape[2]
756 self.Num_Bin = self.spc.shape[1]
756 self.Num_Bin = self.spc.shape[1]
757 self.Num_Chn = self.spc.shape[0]
757 self.Num_Chn = self.spc.shape[0]
758
758
759 VelRange = dataOut.spc_range[2]
759 VelRange = dataOut.spc_range[2]
760
760
761 ''' Se obtiene la constante del RADAR '''
761 ''' Se obtiene la constante del RADAR '''
762
762
763 self.Pt = Pt
763 self.Pt = Pt
764 self.Gt = Gt
764 self.Gt = Gt
765 self.Gr = Gr
765 self.Gr = Gr
766 self.Lambda = Lambda
766 self.Lambda = Lambda
767 self.aL = aL
767 self.aL = aL
768 self.tauW = tauW
768 self.tauW = tauW
769 self.ThetaT = ThetaT
769 self.ThetaT = ThetaT
770 self.ThetaR = ThetaR
770 self.ThetaR = ThetaR
771 self.GSys = 10**(36.63/10) # Ganancia de los LNA 36.63 dB
771 self.GSys = 10**(36.63/10) # Ganancia de los LNA 36.63 dB
772 self.lt = 10**(1.67/10) # Perdida en cables Tx 1.67 dB
772 self.lt = 10**(1.67/10) # Perdida en cables Tx 1.67 dB
773 self.lr = 10**(5.73/10) # Perdida en cables Rx 5.73 dB
773 self.lr = 10**(5.73/10) # Perdida en cables Rx 5.73 dB
774
774
775 Numerator = ( (4*numpy.pi)**3 * aL**2 * 16 * numpy.log(2) )
775 Numerator = ( (4*numpy.pi)**3 * aL**2 * 16 * numpy.log(2) )
776 Denominator = ( Pt * Gt * Gr * Lambda**2 * SPEED_OF_LIGHT * tauW * numpy.pi * ThetaT * ThetaR)
776 Denominator = ( Pt * Gt * Gr * Lambda**2 * SPEED_OF_LIGHT * tauW * numpy.pi * ThetaT * ThetaR)
777 RadarConstant = 10e-26 * Numerator / Denominator #
777 RadarConstant = 10e-26 * Numerator / Denominator #
778 ExpConstant = 10**(40/10) #Constante Experimental
778 ExpConstant = 10**(40/10) #Constante Experimental
779
779
780 SignalPower = numpy.zeros([self.Num_Chn,self.Num_Bin,self.Num_Hei])
780 SignalPower = numpy.zeros([self.Num_Chn,self.Num_Bin,self.Num_Hei])
781 for i in range(self.Num_Chn):
781 for i in range(self.Num_Chn):
782 SignalPower[i,:,:] = self.spc[i,:,:] - dataOut.noise[i]
782 SignalPower[i,:,:] = self.spc[i,:,:] - dataOut.noise[i]
783 SignalPower[numpy.where(SignalPower < 0)] = 1e-20
783 SignalPower[numpy.where(SignalPower < 0)] = 1e-20
784
784
785 SPCmean = numpy.mean(SignalPower, 0)
785 SPCmean = numpy.mean(SignalPower, 0)
786 Pr = SPCmean[:,:]/dataOut.normFactor
786 Pr = SPCmean[:,:]/dataOut.normFactor
787
787
788 # Declaring auxiliary variables
788 # Declaring auxiliary variables
789 Range = dataOut.heightList*1000. #Range in m
789 Range = dataOut.heightList*1000. #Range in m
790 # replicate the heightlist to obtain a matrix [Num_Bin,Num_Hei]
790 # replicate the heightlist to obtain a matrix [Num_Bin,Num_Hei]
791 rMtrx = numpy.transpose(numpy.transpose([dataOut.heightList*1000.] * self.Num_Bin))
791 rMtrx = numpy.transpose(numpy.transpose([dataOut.heightList*1000.] * self.Num_Bin))
792 zMtrx = rMtrx+Altitude
792 zMtrx = rMtrx+Altitude
793 # replicate the VelRange to obtain a matrix [Num_Bin,Num_Hei]
793 # replicate the VelRange to obtain a matrix [Num_Bin,Num_Hei]
794 VelMtrx = numpy.transpose(numpy.tile(VelRange[:-1], (self.Num_Hei,1)))
794 VelMtrx = numpy.transpose(numpy.tile(VelRange[:-1], (self.Num_Hei,1)))
795
795
796 # height dependence to air density Foote and Du Toit (1969)
796 # height dependence to air density Foote and Du Toit (1969)
797 delv_z = 1 + 3.68e-5 * zMtrx + 1.71e-9 * zMtrx**2
797 delv_z = 1 + 3.68e-5 * zMtrx + 1.71e-9 * zMtrx**2
798 VMtrx = VelMtrx / delv_z #Normalized velocity
798 VMtrx = VelMtrx / delv_z #Normalized velocity
799 VMtrx[numpy.where(VMtrx> 9.6)] = numpy.NaN
799 VMtrx[numpy.where(VMtrx> 9.6)] = numpy.NaN
800 # Diameter is related to the fall speed of falling drops
800 # Diameter is related to the fall speed of falling drops
801 D_Vz = -1.667 * numpy.log( 0.9369 - 0.097087 * VMtrx ) # D in [mm]
801 D_Vz = -1.667 * numpy.log( 0.9369 - 0.097087 * VMtrx ) # D in [mm]
802 # Only valid for D>= 0.16 mm
802 # Only valid for D>= 0.16 mm
803 D_Vz[numpy.where(D_Vz < 0.16)] = numpy.NaN
803 D_Vz[numpy.where(D_Vz < 0.16)] = numpy.NaN
804
804
805 #Calculate Radar Reflectivity ETAn
805 #Calculate Radar Reflectivity ETAn
806 ETAn = (RadarConstant *ExpConstant) * Pr * rMtrx**2 #Reflectivity (ETA)
806 ETAn = (RadarConstant *ExpConstant) * Pr * rMtrx**2 #Reflectivity (ETA)
807 ETAd = ETAn * 6.18 * exp( -0.6 * D_Vz ) * delv_z
807 ETAd = ETAn * 6.18 * exp( -0.6 * D_Vz ) * delv_z
808 # Radar Cross Section
808 # Radar Cross Section
809 sigmaD = Km2 * (D_Vz * 1e-3 )**6 * numpy.pi**5 / Lambda**4
809 sigmaD = Km2 * (D_Vz * 1e-3 )**6 * numpy.pi**5 / Lambda**4
810 # Drop Size Distribution
810 # Drop Size Distribution
811 DSD = ETAn / sigmaD
811 DSD = ETAn / sigmaD
812 # Equivalente Reflectivy
812 # Equivalente Reflectivy
813 Ze_eqn = numpy.nansum( DSD * D_Vz**6 ,axis=0)
813 Ze_eqn = numpy.nansum( DSD * D_Vz**6 ,axis=0)
814 Ze_org = numpy.nansum(ETAn * Lambda**4, axis=0) / (1e-18*numpy.pi**5 * Km2) # [mm^6 /m^3]
814 Ze_org = numpy.nansum(ETAn * Lambda**4, axis=0) / (1e-18*numpy.pi**5 * Km2) # [mm^6 /m^3]
815 # RainFall Rate
815 # RainFall Rate
816 RR = 0.0006*numpy.pi * numpy.nansum( D_Vz**3 * DSD * VelMtrx ,0) #mm/hr
816 RR = 0.0006*numpy.pi * numpy.nansum( D_Vz**3 * DSD * VelMtrx ,0) #mm/hr
817
817
818 # Censoring the data
818 # Censoring the data
819 # Removing data with SNRth < 0dB se debe considerar el SNR por canal
819 # Removing data with SNRth < 0dB se debe considerar el SNR por canal
820 SNRth = 10**(SNRdBlimit/10) #-30dB
820 SNRth = 10**(SNRdBlimit/10) #-30dB
821 novalid = numpy.where((dataOut.data_snr[0,:] <SNRth) | (dataOut.data_snr[1,:] <SNRth) | (dataOut.data_snr[2,:] <SNRth)) # AND condition. Maybe OR condition better
821 novalid = numpy.where((dataOut.data_snr[0,:] <SNRth) | (dataOut.data_snr[1,:] <SNRth) | (dataOut.data_snr[2,:] <SNRth)) # AND condition. Maybe OR condition better
822 W = numpy.nanmean(dataOut.data_dop,0)
822 W = numpy.nanmean(dataOut.data_dop,0)
823 W[novalid] = numpy.NaN
823 W[novalid] = numpy.NaN
824 Ze_org[novalid] = numpy.NaN
824 Ze_org[novalid] = numpy.NaN
825 RR[novalid] = numpy.NaN
825 RR[novalid] = numpy.NaN
826
826
827 dataOut.data_output = RR[8]
827 dataOut.data_output = RR[8]
828 dataOut.data_param = numpy.ones([3,self.Num_Hei])
828 dataOut.data_param = numpy.ones([3,self.Num_Hei])
829 dataOut.channelList = [0,1,2]
829 dataOut.channelList = [0,1,2]
830
830
831 dataOut.data_param[0]=10*numpy.log10(Ze_org)
831 dataOut.data_param[0]=10*numpy.log10(Ze_org)
832 dataOut.data_param[1]=-W
832 dataOut.data_param[1]=-W
833 dataOut.data_param[2]=RR
833 dataOut.data_param[2]=RR
834
834
835 # print ('Leaving PrecepitationProc ... ')
835 # print ('Leaving PrecepitationProc ... ')
836 return dataOut
836 return dataOut
837
837
838 def dBZeMODE2(self, dataOut): # Processing for MIRA35C
838 def dBZeMODE2(self, dataOut): # Processing for MIRA35C
839
839
840 NPW = dataOut.NPW
840 NPW = dataOut.NPW
841 COFA = dataOut.COFA
841 COFA = dataOut.COFA
842
842
843 SNR = numpy.array([self.spc[0,:,:] / NPW[0]]) #, self.spc[1,:,:] / NPW[1]])
843 SNR = numpy.array([self.spc[0,:,:] / NPW[0]]) #, self.spc[1,:,:] / NPW[1]])
844 RadarConst = dataOut.RadarConst
844 RadarConst = dataOut.RadarConst
845 #frequency = 34.85*10**9
845 #frequency = 34.85*10**9
846
846
847 ETA = numpy.zeros(([self.Num_Chn ,self.Num_Hei]))
847 ETA = numpy.zeros(([self.Num_Chn ,self.Num_Hei]))
848 data_output = numpy.ones([self.Num_Chn , self.Num_Hei])*numpy.NaN
848 data_output = numpy.ones([self.Num_Chn , self.Num_Hei])*numpy.NaN
849
849
850 ETA = numpy.sum(SNR,1)
850 ETA = numpy.sum(SNR,1)
851
851
852 ETA = numpy.where(ETA != 0. , ETA, numpy.NaN)
852 ETA = numpy.where(ETA != 0. , ETA, numpy.NaN)
853
853
854 Ze = numpy.ones([self.Num_Chn, self.Num_Hei] )
854 Ze = numpy.ones([self.Num_Chn, self.Num_Hei] )
855
855
856 for r in range(self.Num_Hei):
856 for r in range(self.Num_Hei):
857
857
858 Ze[0,r] = ( ETA[0,r] ) * COFA[0,r][0] * RadarConst * ((r/5000.)**2)
858 Ze[0,r] = ( ETA[0,r] ) * COFA[0,r][0] * RadarConst * ((r/5000.)**2)
859 #Ze[1,r] = ( ETA[1,r] ) * COFA[1,r][0] * RadarConst * ((r/5000.)**2)
859 #Ze[1,r] = ( ETA[1,r] ) * COFA[1,r][0] * RadarConst * ((r/5000.)**2)
860
860
861 return Ze
861 return Ze
862
862
863 # def GetRadarConstant(self):
863 # def GetRadarConstant(self):
864 #
864 #
865 # """
865 # """
866 # Constants:
866 # Constants:
867 #
867 #
868 # Pt: Transmission Power dB 5kW 5000
868 # Pt: Transmission Power dB 5kW 5000
869 # Gt: Transmission Gain dB 24.7 dB 295.1209
869 # Gt: Transmission Gain dB 24.7 dB 295.1209
870 # Gr: Reception Gain dB 18.5 dB 70.7945
870 # Gr: Reception Gain dB 18.5 dB 70.7945
871 # Lambda: Wavelenght m 0.6741 m 0.6741
871 # Lambda: Wavelenght m 0.6741 m 0.6741
872 # aL: Attenuation loses dB 4dB 2.5118
872 # aL: Attenuation loses dB 4dB 2.5118
873 # tauW: Width of transmission pulse s 4us 4e-6
873 # tauW: Width of transmission pulse s 4us 4e-6
874 # ThetaT: Transmission antenna bean angle rad 0.1656317 rad 0.1656317
874 # ThetaT: Transmission antenna bean angle rad 0.1656317 rad 0.1656317
875 # ThetaR: Reception antenna beam angle rad 0.36774087 rad 0.36774087
875 # ThetaR: Reception antenna beam angle rad 0.36774087 rad 0.36774087
876 #
876 #
877 # """
877 # """
878 #
878 #
879 # Numerator = ( (4*numpy.pi)**3 * aL**2 * 16 * numpy.log(2) )
879 # Numerator = ( (4*numpy.pi)**3 * aL**2 * 16 * numpy.log(2) )
880 # Denominator = ( Pt * Gt * Gr * Lambda**2 * SPEED_OF_LIGHT * TauW * numpy.pi * ThetaT * TheraR)
880 # Denominator = ( Pt * Gt * Gr * Lambda**2 * SPEED_OF_LIGHT * TauW * numpy.pi * ThetaT * TheraR)
881 # RadarConstant = Numerator / Denominator
881 # RadarConstant = Numerator / Denominator
882 #
882 #
883 # return RadarConstant
883 # return RadarConstant
884
884
885
885
886
886
887 class FullSpectralAnalysis(Operation):
887 class FullSpectralAnalysis(Operation):
888
888
889 """
889 """
890 Function that implements Full Spectral Analysis technique.
890 Function that implements Full Spectral Analysis technique.
891
891
892 Input:
892 Input:
893 self.dataOut.data_pre : SelfSpectra and CrossSpectra data
893 self.dataOut.data_pre : SelfSpectra and CrossSpectra data
894 self.dataOut.groupList : Pairlist of channels
894 self.dataOut.groupList : Pairlist of channels
895 self.dataOut.ChanDist : Physical distance between receivers
895 self.dataOut.ChanDist : Physical distance between receivers
896
896
897
897
898 Output:
898 Output:
899
899
900 self.dataOut.data_output : Zonal wind, Meridional wind, and Vertical wind
900 self.dataOut.data_output : Zonal wind, Meridional wind, and Vertical wind
901
901
902
902
903 Parameters affected: Winds, height range, SNR
903 Parameters affected: Winds, height range, SNR
904
904
905 """
905 """
906 def run(self, dataOut, Xi01=None, Xi02=None, Xi12=None, Eta01=None, Eta02=None, Eta12=None, SNRdBlimit=-30,
906 def run(self, dataOut, Xi01=None, Xi02=None, Xi12=None, Eta01=None, Eta02=None, Eta12=None, SNRdBlimit=-30,
907 minheight=None, maxheight=None, NegativeLimit=None, PositiveLimit=None):
907 minheight=None, maxheight=None, NegativeLimit=None, PositiveLimit=None):
908
908
909 spc = dataOut.data_pre[0].copy()
909 spc = dataOut.data_pre[0].copy()
910 cspc = dataOut.data_pre[1]
910 cspc = dataOut.data_pre[1]
911 nHeights = spc.shape[2]
911 nHeights = spc.shape[2]
912
912
913 # first_height = 0.75 #km (ref: data header 20170822)
913 # first_height = 0.75 #km (ref: data header 20170822)
914 # resolution_height = 0.075 #km
914 # resolution_height = 0.075 #km
915 '''
915 '''
916 finding height range. check this when radar parameters are changed!
916 finding height range. check this when radar parameters are changed!
917 '''
917 '''
918 if maxheight is not None:
918 if maxheight is not None:
919 # range_max = math.ceil((maxheight - first_height) / resolution_height) # theoretical
919 # range_max = math.ceil((maxheight - first_height) / resolution_height) # theoretical
920 range_max = math.ceil(13.26 * maxheight - 3) # empirical, works better
920 range_max = math.ceil(13.26 * maxheight - 3) # empirical, works better
921 else:
921 else:
922 range_max = nHeights
922 range_max = nHeights
923 if minheight is not None:
923 if minheight is not None:
924 # range_min = int((minheight - first_height) / resolution_height) # theoretical
924 # range_min = int((minheight - first_height) / resolution_height) # theoretical
925 range_min = int(13.26 * minheight - 5) # empirical, works better
925 range_min = int(13.26 * minheight - 5) # empirical, works better
926 if range_min < 0:
926 if range_min < 0:
927 range_min = 0
927 range_min = 0
928 else:
928 else:
929 range_min = 0
929 range_min = 0
930
930
931 pairsList = dataOut.groupList
931 pairsList = dataOut.groupList
932 if dataOut.ChanDist is not None :
932 if dataOut.ChanDist is not None :
933 ChanDist = dataOut.ChanDist
933 ChanDist = dataOut.ChanDist
934 else:
934 else:
935 ChanDist = numpy.array([[Xi01, Eta01],[Xi02,Eta02],[Xi12,Eta12]])
935 ChanDist = numpy.array([[Xi01, Eta01],[Xi02,Eta02],[Xi12,Eta12]])
936
936
937 # 4 variables: zonal, meridional, vertical, and average SNR
937 # 4 variables: zonal, meridional, vertical, and average SNR
938 data_param = numpy.zeros([4,nHeights]) * numpy.NaN
938 data_param = numpy.zeros([4,nHeights]) * numpy.NaN
939 velocityX = numpy.zeros([nHeights]) * numpy.NaN
939 velocityX = numpy.zeros([nHeights]) * numpy.NaN
940 velocityY = numpy.zeros([nHeights]) * numpy.NaN
940 velocityY = numpy.zeros([nHeights]) * numpy.NaN
941 velocityZ = numpy.zeros([nHeights]) * numpy.NaN
941 velocityZ = numpy.zeros([nHeights]) * numpy.NaN
942
942
943 dbSNR = 10*numpy.log10(numpy.average(dataOut.data_snr,0))
943 dbSNR = 10*numpy.log10(numpy.average(dataOut.data_snr,0))
944
944
945 '''***********************************************WIND ESTIMATION**************************************'''
945 '''***********************************************WIND ESTIMATION**************************************'''
946 for Height in range(nHeights):
946 for Height in range(nHeights):
947
947
948 if Height >= range_min and Height < range_max:
948 if Height >= range_min and Height < range_max:
949 # error_code will be useful in future analysis
949 # error_code will be useful in future analysis
950 [Vzon,Vmer,Vver, error_code] = self.WindEstimation(spc[:,:,Height], cspc[:,:,Height], pairsList,
950 [Vzon,Vmer,Vver, error_code] = self.WindEstimation(spc[:,:,Height], cspc[:,:,Height], pairsList,
951 ChanDist, Height, dataOut.noise, dataOut.spc_range, dbSNR[Height], SNRdBlimit, NegativeLimit, PositiveLimit,dataOut.frequency)
951 ChanDist, Height, dataOut.noise, dataOut.spc_range, dbSNR[Height], SNRdBlimit, NegativeLimit, PositiveLimit,dataOut.frequency)
952
952
953 if abs(Vzon) < 100. and abs(Vmer) < 100.:
953 if abs(Vzon) < 100. and abs(Vmer) < 100.:
954 velocityX[Height] = Vzon
954 velocityX[Height] = Vzon
955 velocityY[Height] = -Vmer
955 velocityY[Height] = -Vmer
956 velocityZ[Height] = Vver
956 velocityZ[Height] = Vver
957
957
958 # Censoring data with SNR threshold
958 # Censoring data with SNR threshold
959 dbSNR [dbSNR < SNRdBlimit] = numpy.NaN
959 dbSNR [dbSNR < SNRdBlimit] = numpy.NaN
960
960
961 data_param[0] = velocityX
961 data_param[0] = velocityX
962 data_param[1] = velocityY
962 data_param[1] = velocityY
963 data_param[2] = velocityZ
963 data_param[2] = velocityZ
964 data_param[3] = dbSNR
964 data_param[3] = dbSNR
965 dataOut.data_param = data_param
965 dataOut.data_param = data_param
966 return dataOut
966 return dataOut
967
967
968 def moving_average(self,x, N=2):
968 def moving_average(self,x, N=2):
969 """ convolution for smoothenig data. note that last N-1 values are convolution with zeroes """
969 """ convolution for smoothenig data. note that last N-1 values are convolution with zeroes """
970 return numpy.convolve(x, numpy.ones((N,))/N)[(N-1):]
970 return numpy.convolve(x, numpy.ones((N,))/N)[(N-1):]
971
971
972 def gaus(self,xSamples,Amp,Mu,Sigma):
972 def gaus(self,xSamples,Amp,Mu,Sigma):
973 return Amp * numpy.exp(-0.5*((xSamples - Mu)/Sigma)**2)
973 return Amp * numpy.exp(-0.5*((xSamples - Mu)/Sigma)**2)
974
974
975 def Moments(self, ySamples, xSamples):
975 def Moments(self, ySamples, xSamples):
976 Power = numpy.nanmean(ySamples) # Power, 0th Moment
976 Power = numpy.nanmean(ySamples) # Power, 0th Moment
977 yNorm = ySamples / numpy.nansum(ySamples)
977 yNorm = ySamples / numpy.nansum(ySamples)
978 RadVel = numpy.nansum(xSamples * yNorm) # Radial Velocity, 1st Moment
978 RadVel = numpy.nansum(xSamples * yNorm) # Radial Velocity, 1st Moment
979 Sigma2 = numpy.nansum(yNorm * (xSamples - RadVel)**2) # Spectral Width, 2nd Moment
979 Sigma2 = numpy.nansum(yNorm * (xSamples - RadVel)**2) # Spectral Width, 2nd Moment
980 StdDev = numpy.sqrt(numpy.abs(Sigma2)) # Desv. Estandar, Ancho espectral
980 StdDev = numpy.sqrt(numpy.abs(Sigma2)) # Desv. Estandar, Ancho espectral
981 return numpy.array([Power,RadVel,StdDev])
981 return numpy.array([Power,RadVel,StdDev])
982
982
983 def StopWindEstimation(self, error_code):
983 def StopWindEstimation(self, error_code):
984 Vzon = numpy.NaN
984 Vzon = numpy.NaN
985 Vmer = numpy.NaN
985 Vmer = numpy.NaN
986 Vver = numpy.NaN
986 Vver = numpy.NaN
987 return Vzon, Vmer, Vver, error_code
987 return Vzon, Vmer, Vver, error_code
988
988
989 def AntiAliasing(self, interval, maxstep):
989 def AntiAliasing(self, interval, maxstep):
990 """
990 """
991 function to prevent errors from aliased values when computing phaseslope
991 function to prevent errors from aliased values when computing phaseslope
992 """
992 """
993 antialiased = numpy.zeros(len(interval))
993 antialiased = numpy.zeros(len(interval))
994 copyinterval = interval.copy()
994 copyinterval = interval.copy()
995
995
996 antialiased[0] = copyinterval[0]
996 antialiased[0] = copyinterval[0]
997
997
998 for i in range(1,len(antialiased)):
998 for i in range(1,len(antialiased)):
999 step = interval[i] - interval[i-1]
999 step = interval[i] - interval[i-1]
1000 if step > maxstep:
1000 if step > maxstep:
1001 copyinterval -= 2*numpy.pi
1001 copyinterval -= 2*numpy.pi
1002 antialiased[i] = copyinterval[i]
1002 antialiased[i] = copyinterval[i]
1003 elif step < maxstep*(-1):
1003 elif step < maxstep*(-1):
1004 copyinterval += 2*numpy.pi
1004 copyinterval += 2*numpy.pi
1005 antialiased[i] = copyinterval[i]
1005 antialiased[i] = copyinterval[i]
1006 else:
1006 else:
1007 antialiased[i] = copyinterval[i].copy()
1007 antialiased[i] = copyinterval[i].copy()
1008
1008
1009 return antialiased
1009 return antialiased
1010
1010
1011 def WindEstimation(self, spc, cspc, pairsList, ChanDist, Height, noise, AbbsisaRange, dbSNR, SNRlimit, NegativeLimit, PositiveLimit, radfreq):
1011 def WindEstimation(self, spc, cspc, pairsList, ChanDist, Height, noise, AbbsisaRange, dbSNR, SNRlimit, NegativeLimit, PositiveLimit, radfreq):
1012 """
1012 """
1013 Function that Calculates Zonal, Meridional and Vertical wind velocities.
1013 Function that Calculates Zonal, Meridional and Vertical wind velocities.
1014 Initial Version by E. Bocanegra updated by J. Zibell until Nov. 2019.
1014 Initial Version by E. Bocanegra updated by J. Zibell until Nov. 2019.
1015
1015
1016 Input:
1016 Input:
1017 spc, cspc : self spectra and cross spectra data. In Briggs notation something like S_i*(S_i)_conj, (S_j)_conj respectively.
1017 spc, cspc : self spectra and cross spectra data. In Briggs notation something like S_i*(S_i)_conj, (S_j)_conj respectively.
1018 pairsList : Pairlist of channels
1018 pairsList : Pairlist of channels
1019 ChanDist : array of xi_ij and eta_ij
1019 ChanDist : array of xi_ij and eta_ij
1020 Height : height at which data is processed
1020 Height : height at which data is processed
1021 noise : noise in [channels] format for specific height
1021 noise : noise in [channels] format for specific height
1022 Abbsisarange : range of the frequencies or velocities
1022 Abbsisarange : range of the frequencies or velocities
1023 dbSNR, SNRlimit : signal to noise ratio in db, lower limit
1023 dbSNR, SNRlimit : signal to noise ratio in db, lower limit
1024
1024
1025 Output:
1025 Output:
1026 Vzon, Vmer, Vver : wind velocities
1026 Vzon, Vmer, Vver : wind velocities
1027 error_code : int that states where code is terminated
1027 error_code : int that states where code is terminated
1028
1028
1029 0 : no error detected
1029 0 : no error detected
1030 1 : Gaussian of mean spc exceeds widthlimit
1030 1 : Gaussian of mean spc exceeds widthlimit
1031 2 : no Gaussian of mean spc found
1031 2 : no Gaussian of mean spc found
1032 3 : SNR to low or velocity to high -> prec. e.g.
1032 3 : SNR to low or velocity to high -> prec. e.g.
1033 4 : at least one Gaussian of cspc exceeds widthlimit
1033 4 : at least one Gaussian of cspc exceeds widthlimit
1034 5 : zero out of three cspc Gaussian fits converged
1034 5 : zero out of three cspc Gaussian fits converged
1035 6 : phase slope fit could not be found
1035 6 : phase slope fit could not be found
1036 7 : arrays used to fit phase have different length
1036 7 : arrays used to fit phase have different length
1037 8 : frequency range is either too short (len <= 5) or very long (> 30% of cspc)
1037 8 : frequency range is either too short (len <= 5) or very long (> 30% of cspc)
1038
1038
1039 """
1039 """
1040
1040
1041 error_code = 0
1041 error_code = 0
1042
1042
1043 nChan = spc.shape[0]
1043 nChan = spc.shape[0]
1044 nProf = spc.shape[1]
1044 nProf = spc.shape[1]
1045 nPair = cspc.shape[0]
1045 nPair = cspc.shape[0]
1046
1046
1047 SPC_Samples = numpy.zeros([nChan, nProf]) # for normalized spc values for one height
1047 SPC_Samples = numpy.zeros([nChan, nProf]) # for normalized spc values for one height
1048 CSPC_Samples = numpy.zeros([nPair, nProf], dtype=numpy.complex_) # for normalized cspc values
1048 CSPC_Samples = numpy.zeros([nPair, nProf], dtype=numpy.complex_) # for normalized cspc values
1049 phase = numpy.zeros([nPair, nProf]) # phase between channels
1049 phase = numpy.zeros([nPair, nProf]) # phase between channels
1050 PhaseSlope = numpy.zeros(nPair) # slope of the phases, channelwise
1050 PhaseSlope = numpy.zeros(nPair) # slope of the phases, channelwise
1051 PhaseInter = numpy.zeros(nPair) # intercept to the slope of the phases, channelwise
1051 PhaseInter = numpy.zeros(nPair) # intercept to the slope of the phases, channelwise
1052 xFrec = AbbsisaRange[0][:-1] # frequency range
1052 xFrec = AbbsisaRange[0][:-1] # frequency range
1053 xVel = AbbsisaRange[2][:-1] # velocity range
1053 xVel = AbbsisaRange[2][:-1] # velocity range
1054 xSamples = xFrec # the frequency range is taken
1054 xSamples = xFrec # the frequency range is taken
1055 delta_x = xSamples[1] - xSamples[0] # delta_f or delta_x
1055 delta_x = xSamples[1] - xSamples[0] # delta_f or delta_x
1056
1056
1057 # only consider velocities with in NegativeLimit and PositiveLimit
1057 # only consider velocities with in NegativeLimit and PositiveLimit
1058 if (NegativeLimit is None):
1058 if (NegativeLimit is None):
1059 NegativeLimit = numpy.min(xVel)
1059 NegativeLimit = numpy.min(xVel)
1060 if (PositiveLimit is None):
1060 if (PositiveLimit is None):
1061 PositiveLimit = numpy.max(xVel)
1061 PositiveLimit = numpy.max(xVel)
1062 xvalid = numpy.where((xVel > NegativeLimit) & (xVel < PositiveLimit))
1062 xvalid = numpy.where((xVel > NegativeLimit) & (xVel < PositiveLimit))
1063 xSamples_zoom = xSamples[xvalid]
1063 xSamples_zoom = xSamples[xvalid]
1064
1064
1065 '''Getting Eij and Nij'''
1065 '''Getting Eij and Nij'''
1066 Xi01, Xi02, Xi12 = ChanDist[:,0]
1066 Xi01, Xi02, Xi12 = ChanDist[:,0]
1067 Eta01, Eta02, Eta12 = ChanDist[:,1]
1067 Eta01, Eta02, Eta12 = ChanDist[:,1]
1068
1068
1069 # spwd limit - updated by D. Scipión 30.03.2021
1069 # spwd limit - updated by D. Scipión 30.03.2021
1070 widthlimit = 10
1070 widthlimit = 10
1071 '''************************* SPC is normalized ********************************'''
1071 '''************************* SPC is normalized ********************************'''
1072 spc_norm = spc.copy()
1072 spc_norm = spc.copy()
1073 # For each channel
1073 # For each channel
1074 for i in range(nChan):
1074 for i in range(nChan):
1075 spc_sub = spc_norm[i,:] - noise[i] # only the signal power
1075 spc_sub = spc_norm[i,:] - noise[i] # only the signal power
1076 SPC_Samples[i] = spc_sub / (numpy.nansum(spc_sub) * delta_x)
1076 SPC_Samples[i] = spc_sub / (numpy.nansum(spc_sub) * delta_x)
1077
1077
1078 '''********************** FITTING MEAN SPC GAUSSIAN **********************'''
1078 '''********************** FITTING MEAN SPC GAUSSIAN **********************'''
1079
1079
1080 """ the gaussian of the mean: first subtract noise, then normalize. this is legal because
1080 """ the gaussian of the mean: first subtract noise, then normalize. this is legal because
1081 you only fit the curve and don't need the absolute value of height for calculation,
1081 you only fit the curve and don't need the absolute value of height for calculation,
1082 only for estimation of width. for normalization of cross spectra, you need initial,
1082 only for estimation of width. for normalization of cross spectra, you need initial,
1083 unnormalized self-spectra With noise.
1083 unnormalized self-spectra With noise.
1084
1084
1085 Technically, you don't even need to normalize the self-spectra, as you only need the
1085 Technically, you don't even need to normalize the self-spectra, as you only need the
1086 width of the peak. However, it was left this way. Note that the normalization has a flaw:
1086 width of the peak. However, it was left this way. Note that the normalization has a flaw:
1087 due to subtraction of the noise, some values are below zero. Raw "spc" values should be
1087 due to subtraction of the noise, some values are below zero. Raw "spc" values should be
1088 >= 0, as it is the modulus squared of the signals (complex * it's conjugate)
1088 >= 0, as it is the modulus squared of the signals (complex * it's conjugate)
1089 """
1089 """
1090 # initial conditions
1090 # initial conditions
1091 popt = [1e-10,0,1e-10]
1091 popt = [1e-10,0,1e-10]
1092 # Spectra average
1092 # Spectra average
1093 SPCMean = numpy.average(SPC_Samples,0)
1093 SPCMean = numpy.average(SPC_Samples,0)
1094 # Moments in frequency
1094 # Moments in frequency
1095 SPCMoments = self.Moments(SPCMean[xvalid], xSamples_zoom)
1095 SPCMoments = self.Moments(SPCMean[xvalid], xSamples_zoom)
1096
1096
1097 # Gauss Fit SPC in frequency domain
1097 # Gauss Fit SPC in frequency domain
1098 if dbSNR > SNRlimit: # only if SNR > SNRth
1098 if dbSNR > SNRlimit: # only if SNR > SNRth
1099 try:
1099 try:
1100 popt,pcov = curve_fit(self.gaus,xSamples_zoom,SPCMean[xvalid],p0=SPCMoments)
1100 popt,pcov = curve_fit(self.gaus,xSamples_zoom,SPCMean[xvalid],p0=SPCMoments)
1101 if popt[2] <= 0 or popt[2] > widthlimit: # CONDITION
1101 if popt[2] <= 0 or popt[2] > widthlimit: # CONDITION
1102 return self.StopWindEstimation(error_code = 1)
1102 return self.StopWindEstimation(error_code = 1)
1103 FitGauss = self.gaus(xSamples_zoom,*popt)
1103 FitGauss = self.gaus(xSamples_zoom,*popt)
1104 except :#RuntimeError:
1104 except :#RuntimeError:
1105 return self.StopWindEstimation(error_code = 2)
1105 return self.StopWindEstimation(error_code = 2)
1106 else:
1106 else:
1107 return self.StopWindEstimation(error_code = 3)
1107 return self.StopWindEstimation(error_code = 3)
1108
1108
1109 '''***************************** CSPC Normalization *************************
1109 '''***************************** CSPC Normalization *************************
1110 The Spc spectra are used to normalize the crossspectra. Peaks from precipitation
1110 The Spc spectra are used to normalize the crossspectra. Peaks from precipitation
1111 influence the norm which is not desired. First, a range is identified where the
1111 influence the norm which is not desired. First, a range is identified where the
1112 wind peak is estimated -> sum_wind is sum of those frequencies. Next, the area
1112 wind peak is estimated -> sum_wind is sum of those frequencies. Next, the area
1113 around it gets cut off and values replaced by mean determined by the boundary
1113 around it gets cut off and values replaced by mean determined by the boundary
1114 data -> sum_noise (spc is not normalized here, thats why the noise is important)
1114 data -> sum_noise (spc is not normalized here, thats why the noise is important)
1115
1115
1116 The sums are then added and multiplied by range/datapoints, because you need
1116 The sums are then added and multiplied by range/datapoints, because you need
1117 an integral and not a sum for normalization.
1117 an integral and not a sum for normalization.
1118
1118
1119 A norm is found according to Briggs 92.
1119 A norm is found according to Briggs 92.
1120 '''
1120 '''
1121 # for each pair
1121 # for each pair
1122 for i in range(nPair):
1122 for i in range(nPair):
1123 cspc_norm = cspc[i,:].copy()
1123 cspc_norm = cspc[i,:].copy()
1124 chan_index0 = pairsList[i][0]
1124 chan_index0 = pairsList[i][0]
1125 chan_index1 = pairsList[i][1]
1125 chan_index1 = pairsList[i][1]
1126 CSPC_Samples[i] = cspc_norm / (numpy.sqrt(numpy.nansum(spc_norm[chan_index0])*numpy.nansum(spc_norm[chan_index1])) * delta_x)
1126 CSPC_Samples[i] = cspc_norm / (numpy.sqrt(numpy.nansum(spc_norm[chan_index0])*numpy.nansum(spc_norm[chan_index1])) * delta_x)
1127 phase[i] = numpy.arctan2(CSPC_Samples[i].imag, CSPC_Samples[i].real)
1127 phase[i] = numpy.arctan2(CSPC_Samples[i].imag, CSPC_Samples[i].real)
1128
1128
1129 CSPCmoments = numpy.vstack([self.Moments(numpy.abs(CSPC_Samples[0,xvalid]), xSamples_zoom),
1129 CSPCmoments = numpy.vstack([self.Moments(numpy.abs(CSPC_Samples[0,xvalid]), xSamples_zoom),
1130 self.Moments(numpy.abs(CSPC_Samples[1,xvalid]), xSamples_zoom),
1130 self.Moments(numpy.abs(CSPC_Samples[1,xvalid]), xSamples_zoom),
1131 self.Moments(numpy.abs(CSPC_Samples[2,xvalid]), xSamples_zoom)])
1131 self.Moments(numpy.abs(CSPC_Samples[2,xvalid]), xSamples_zoom)])
1132
1132
1133 popt01, popt02, popt12 = [1e-10,0,1e-10], [1e-10,0,1e-10] ,[1e-10,0,1e-10]
1133 popt01, popt02, popt12 = [1e-10,0,1e-10], [1e-10,0,1e-10] ,[1e-10,0,1e-10]
1134 FitGauss01, FitGauss02, FitGauss12 = numpy.zeros(len(xSamples)), numpy.zeros(len(xSamples)), numpy.zeros(len(xSamples))
1134 FitGauss01, FitGauss02, FitGauss12 = numpy.zeros(len(xSamples)), numpy.zeros(len(xSamples)), numpy.zeros(len(xSamples))
1135
1135
1136 '''*******************************FIT GAUSS CSPC************************************'''
1136 '''*******************************FIT GAUSS CSPC************************************'''
1137 try:
1137 try:
1138 popt01,pcov = curve_fit(self.gaus,xSamples_zoom,numpy.abs(CSPC_Samples[0][xvalid]),p0=CSPCmoments[0])
1138 popt01,pcov = curve_fit(self.gaus,xSamples_zoom,numpy.abs(CSPC_Samples[0][xvalid]),p0=CSPCmoments[0])
1139 if popt01[2] > widthlimit: # CONDITION
1139 if popt01[2] > widthlimit: # CONDITION
1140 return self.StopWindEstimation(error_code = 4)
1140 return self.StopWindEstimation(error_code = 4)
1141 popt02,pcov = curve_fit(self.gaus,xSamples_zoom,numpy.abs(CSPC_Samples[1][xvalid]),p0=CSPCmoments[1])
1141 popt02,pcov = curve_fit(self.gaus,xSamples_zoom,numpy.abs(CSPC_Samples[1][xvalid]),p0=CSPCmoments[1])
1142 if popt02[2] > widthlimit: # CONDITION
1142 if popt02[2] > widthlimit: # CONDITION
1143 return self.StopWindEstimation(error_code = 4)
1143 return self.StopWindEstimation(error_code = 4)
1144 popt12,pcov = curve_fit(self.gaus,xSamples_zoom,numpy.abs(CSPC_Samples[2][xvalid]),p0=CSPCmoments[2])
1144 popt12,pcov = curve_fit(self.gaus,xSamples_zoom,numpy.abs(CSPC_Samples[2][xvalid]),p0=CSPCmoments[2])
1145 if popt12[2] > widthlimit: # CONDITION
1145 if popt12[2] > widthlimit: # CONDITION
1146 return self.StopWindEstimation(error_code = 4)
1146 return self.StopWindEstimation(error_code = 4)
1147
1147
1148 FitGauss01 = self.gaus(xSamples_zoom, *popt01)
1148 FitGauss01 = self.gaus(xSamples_zoom, *popt01)
1149 FitGauss02 = self.gaus(xSamples_zoom, *popt02)
1149 FitGauss02 = self.gaus(xSamples_zoom, *popt02)
1150 FitGauss12 = self.gaus(xSamples_zoom, *popt12)
1150 FitGauss12 = self.gaus(xSamples_zoom, *popt12)
1151 except:
1151 except:
1152 return self.StopWindEstimation(error_code = 5)
1152 return self.StopWindEstimation(error_code = 5)
1153
1153
1154
1154
1155 '''************* Getting Fij ***************'''
1155 '''************* Getting Fij ***************'''
1156 # x-axis point of the gaussian where the center is located from GaussFit of spectra
1156 # x-axis point of the gaussian where the center is located from GaussFit of spectra
1157 GaussCenter = popt[1]
1157 GaussCenter = popt[1]
1158 ClosestCenter = xSamples_zoom[numpy.abs(xSamples_zoom-GaussCenter).argmin()]
1158 ClosestCenter = xSamples_zoom[numpy.abs(xSamples_zoom-GaussCenter).argmin()]
1159 PointGauCenter = numpy.where(xSamples_zoom==ClosestCenter)[0][0]
1159 PointGauCenter = numpy.where(xSamples_zoom==ClosestCenter)[0][0]
1160
1160
1161 # Point where e^-1 is located in the gaussian
1161 # Point where e^-1 is located in the gaussian
1162 PeMinus1 = numpy.max(FitGauss) * numpy.exp(-1)
1162 PeMinus1 = numpy.max(FitGauss) * numpy.exp(-1)
1163 FijClosest = FitGauss[numpy.abs(FitGauss-PeMinus1).argmin()] # The closest point to"Peminus1" in "FitGauss"
1163 FijClosest = FitGauss[numpy.abs(FitGauss-PeMinus1).argmin()] # The closest point to"Peminus1" in "FitGauss"
1164 PointFij = numpy.where(FitGauss==FijClosest)[0][0]
1164 PointFij = numpy.where(FitGauss==FijClosest)[0][0]
1165 Fij = numpy.abs(xSamples_zoom[PointFij] - xSamples_zoom[PointGauCenter])
1165 Fij = numpy.abs(xSamples_zoom[PointFij] - xSamples_zoom[PointGauCenter])
1166
1166
1167 '''********** Taking frequency ranges from mean SPCs **********'''
1167 '''********** Taking frequency ranges from mean SPCs **********'''
1168 GauWidth = popt[2] * 3/2 # Bandwidth of Gau01
1168 GauWidth = popt[2] * 3/2 # Bandwidth of Gau01
1169 Range = numpy.empty(2)
1169 Range = numpy.empty(2)
1170 Range[0] = GaussCenter - GauWidth
1170 Range[0] = GaussCenter - GauWidth
1171 Range[1] = GaussCenter + GauWidth
1171 Range[1] = GaussCenter + GauWidth
1172 # Point in x-axis where the bandwidth is located (min:max)
1172 # Point in x-axis where the bandwidth is located (min:max)
1173 ClosRangeMin = xSamples_zoom[numpy.abs(xSamples_zoom-Range[0]).argmin()]
1173 ClosRangeMin = xSamples_zoom[numpy.abs(xSamples_zoom-Range[0]).argmin()]
1174 ClosRangeMax = xSamples_zoom[numpy.abs(xSamples_zoom-Range[1]).argmin()]
1174 ClosRangeMax = xSamples_zoom[numpy.abs(xSamples_zoom-Range[1]).argmin()]
1175 PointRangeMin = numpy.where(xSamples_zoom==ClosRangeMin)[0][0]
1175 PointRangeMin = numpy.where(xSamples_zoom==ClosRangeMin)[0][0]
1176 PointRangeMax = numpy.where(xSamples_zoom==ClosRangeMax)[0][0]
1176 PointRangeMax = numpy.where(xSamples_zoom==ClosRangeMax)[0][0]
1177 Range = numpy.array([ PointRangeMin, PointRangeMax ])
1177 Range = numpy.array([ PointRangeMin, PointRangeMax ])
1178 FrecRange = xSamples_zoom[ Range[0] : Range[1] ]
1178 FrecRange = xSamples_zoom[ Range[0] : Range[1] ]
1179
1179
1180 '''************************** Getting Phase Slope ***************************'''
1180 '''************************** Getting Phase Slope ***************************'''
1181 for i in range(nPair):
1181 for i in range(nPair):
1182 if len(FrecRange) > 5:
1182 if len(FrecRange) > 5:
1183 PhaseRange = phase[i, xvalid[0][Range[0]:Range[1]]].copy()
1183 PhaseRange = phase[i, xvalid[0][Range[0]:Range[1]]].copy()
1184 mask = ~numpy.isnan(FrecRange) & ~numpy.isnan(PhaseRange)
1184 mask = ~numpy.isnan(FrecRange) & ~numpy.isnan(PhaseRange)
1185 if len(FrecRange) == len(PhaseRange):
1185 if len(FrecRange) == len(PhaseRange):
1186 try:
1186 try:
1187 slope, intercept, _, _, _ = stats.linregress(FrecRange[mask], self.AntiAliasing(PhaseRange[mask], 4.5))
1187 slope, intercept, _, _, _ = stats.linregress(FrecRange[mask], self.AntiAliasing(PhaseRange[mask], 4.5))
1188 PhaseSlope[i] = slope
1188 PhaseSlope[i] = slope
1189 PhaseInter[i] = intercept
1189 PhaseInter[i] = intercept
1190 except:
1190 except:
1191 return self.StopWindEstimation(error_code = 6)
1191 return self.StopWindEstimation(error_code = 6)
1192 else:
1192 else:
1193 return self.StopWindEstimation(error_code = 7)
1193 return self.StopWindEstimation(error_code = 7)
1194 else:
1194 else:
1195 return self.StopWindEstimation(error_code = 8)
1195 return self.StopWindEstimation(error_code = 8)
1196
1196
1197 '''*** Constants A-H correspond to the convention as in Briggs and Vincent 1992 ***'''
1197 '''*** Constants A-H correspond to the convention as in Briggs and Vincent 1992 ***'''
1198
1198
1199 '''Getting constant C'''
1199 '''Getting constant C'''
1200 cC=(Fij*numpy.pi)**2
1200 cC=(Fij*numpy.pi)**2
1201
1201
1202 '''****** Getting constants F and G ******'''
1202 '''****** Getting constants F and G ******'''
1203 MijEijNij = numpy.array([[Xi02,Eta02], [Xi12,Eta12]])
1203 MijEijNij = numpy.array([[Xi02,Eta02], [Xi12,Eta12]])
1204 # MijEijNij = numpy.array([[Xi01,Eta01], [Xi02,Eta02], [Xi12,Eta12]])
1204 # MijEijNij = numpy.array([[Xi01,Eta01], [Xi02,Eta02], [Xi12,Eta12]])
1205 # MijResult0 = (-PhaseSlope[0] * cC) / (2*numpy.pi)
1205 # MijResult0 = (-PhaseSlope[0] * cC) / (2*numpy.pi)
1206 MijResult1 = (-PhaseSlope[1] * cC) / (2*numpy.pi)
1206 MijResult1 = (-PhaseSlope[1] * cC) / (2*numpy.pi)
1207 MijResult2 = (-PhaseSlope[2] * cC) / (2*numpy.pi)
1207 MijResult2 = (-PhaseSlope[2] * cC) / (2*numpy.pi)
1208 # MijResults = numpy.array([MijResult0, MijResult1, MijResult2])
1208 # MijResults = numpy.array([MijResult0, MijResult1, MijResult2])
1209 MijResults = numpy.array([MijResult1, MijResult2])
1209 MijResults = numpy.array([MijResult1, MijResult2])
1210 (cF,cG) = numpy.linalg.solve(MijEijNij, MijResults)
1210 (cF,cG) = numpy.linalg.solve(MijEijNij, MijResults)
1211
1211
1212 '''****** Getting constants A, B and H ******'''
1212 '''****** Getting constants A, B and H ******'''
1213 W01 = numpy.nanmax( FitGauss01 )
1213 W01 = numpy.nanmax( FitGauss01 )
1214 W02 = numpy.nanmax( FitGauss02 )
1214 W02 = numpy.nanmax( FitGauss02 )
1215 W12 = numpy.nanmax( FitGauss12 )
1215 W12 = numpy.nanmax( FitGauss12 )
1216
1216
1217 WijResult01 = ((cF * Xi01 + cG * Eta01)**2)/cC - numpy.log(W01 / numpy.sqrt(numpy.pi / cC))
1217 WijResult01 = ((cF * Xi01 + cG * Eta01)**2)/cC - numpy.log(W01 / numpy.sqrt(numpy.pi / cC))
1218 WijResult02 = ((cF * Xi02 + cG * Eta02)**2)/cC - numpy.log(W02 / numpy.sqrt(numpy.pi / cC))
1218 WijResult02 = ((cF * Xi02 + cG * Eta02)**2)/cC - numpy.log(W02 / numpy.sqrt(numpy.pi / cC))
1219 WijResult12 = ((cF * Xi12 + cG * Eta12)**2)/cC - numpy.log(W12 / numpy.sqrt(numpy.pi / cC))
1219 WijResult12 = ((cF * Xi12 + cG * Eta12)**2)/cC - numpy.log(W12 / numpy.sqrt(numpy.pi / cC))
1220 WijResults = numpy.array([WijResult01, WijResult02, WijResult12])
1220 WijResults = numpy.array([WijResult01, WijResult02, WijResult12])
1221
1221
1222 WijEijNij = numpy.array([ [Xi01**2, Eta01**2, 2*Xi01*Eta01] , [Xi02**2, Eta02**2, 2*Xi02*Eta02] , [Xi12**2, Eta12**2, 2*Xi12*Eta12] ])
1222 WijEijNij = numpy.array([ [Xi01**2, Eta01**2, 2*Xi01*Eta01] , [Xi02**2, Eta02**2, 2*Xi02*Eta02] , [Xi12**2, Eta12**2, 2*Xi12*Eta12] ])
1223 (cA,cB,cH) = numpy.linalg.solve(WijEijNij, WijResults)
1223 (cA,cB,cH) = numpy.linalg.solve(WijEijNij, WijResults)
1224
1224
1225 VxVy = numpy.array([[cA,cH],[cH,cB]])
1225 VxVy = numpy.array([[cA,cH],[cH,cB]])
1226 VxVyResults = numpy.array([-cF,-cG])
1226 VxVyResults = numpy.array([-cF,-cG])
1227 (Vmer,Vzon) = numpy.linalg.solve(VxVy, VxVyResults)
1227 (Vmer,Vzon) = numpy.linalg.solve(VxVy, VxVyResults)
1228 Vver = -SPCMoments[1]*SPEED_OF_LIGHT/(2*radfreq)
1228 Vver = -SPCMoments[1]*SPEED_OF_LIGHT/(2*radfreq)
1229 error_code = 0
1229 error_code = 0
1230
1230
1231 return Vzon, Vmer, Vver, error_code
1231 return Vzon, Vmer, Vver, error_code
1232
1232
1233 class SpectralMoments(Operation):
1233 class SpectralMoments(Operation):
1234
1234
1235 '''
1235 '''
1236 Function SpectralMoments()
1236 Function SpectralMoments()
1237
1237
1238 Calculates moments (power, mean, standard deviation) and SNR of the signal
1238 Calculates moments (power, mean, standard deviation) and SNR of the signal
1239
1239
1240 Type of dataIn: Spectra
1240 Type of dataIn: Spectra
1241
1241
1242 Configuration Parameters:
1242 Configuration Parameters:
1243
1243
1244 dirCosx : Cosine director in X axis
1244 dirCosx : Cosine director in X axis
1245 dirCosy : Cosine director in Y axis
1245 dirCosy : Cosine director in Y axis
1246
1246
1247 elevation :
1247 elevation :
1248 azimuth :
1248 azimuth :
1249
1249
1250 Input:
1250 Input:
1251 channelList : simple channel list to select e.g. [2,3,7]
1251 channelList : simple channel list to select e.g. [2,3,7]
1252 self.dataOut.data_pre : Spectral data
1252 self.dataOut.data_pre : Spectral data
1253 self.dataOut.abscissaList : List of frequencies
1253 self.dataOut.abscissaList : List of frequencies
1254 self.dataOut.noise : Noise level per channel
1254 self.dataOut.noise : Noise level per channel
1255
1255
1256 Affected:
1256 Affected:
1257 self.dataOut.moments : Parameters per channel
1257 self.dataOut.moments : Parameters per channel
1258 self.dataOut.data_snr : SNR per channel
1258 self.dataOut.data_snr : SNR per channel
1259
1259
1260 '''
1260 '''
1261
1261
1262 def run(self, dataOut,wradar=False):
1262 def run(self, dataOut,wradar=False):
1263
1263
1264 data = dataOut.data_pre[0]
1264 data = dataOut.data_pre[0]
1265 absc = dataOut.abscissaList[:-1]
1265 absc = dataOut.abscissaList[:-1]
1266 noise = dataOut.noise
1266 noise = dataOut.noise
1267 nChannel = data.shape[0]
1267 nChannel = data.shape[0]
1268 data_param = numpy.zeros((nChannel, 4, data.shape[2]))
1268 data_param = numpy.zeros((nChannel, 4, data.shape[2]))
1269
1269
1270 for ind in range(nChannel):
1270 for ind in range(nChannel):
1271 data_param[ind,:,:] = self.__calculateMoments( data[ind,:,:] , absc , noise[ind],wradar=wradar )
1271 data_param[ind,:,:] = self.__calculateMoments( data[ind,:,:] , absc , noise[ind],wradar=wradar )
1272
1272
1273 dataOut.moments = data_param[:,1:,:]
1273 dataOut.moments = data_param[:,1:,:]
1274 dataOut.data_snr = data_param[:,0]
1274 dataOut.data_snr = data_param[:,0]
1275 dataOut.data_pow = data_param[:,1]
1275 dataOut.data_pow = data_param[:,1]
1276 dataOut.data_dop = data_param[:,2]
1276 dataOut.data_dop = data_param[:,2]
1277 dataOut.data_width = data_param[:,3]
1277 dataOut.data_width = data_param[:,3]
1278 return dataOut
1278 return dataOut
1279
1279
1280 def __calculateMoments(self, oldspec, oldfreq, n0,
1280 def __calculateMoments(self, oldspec, oldfreq, n0,
1281 nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None,wradar=None):
1281 nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None,wradar=None):
1282
1282
1283 if (nicoh is None): nicoh = 1
1283 if (nicoh is None): nicoh = 1
1284 if (graph is None): graph = 0
1284 if (graph is None): graph = 0
1285 if (smooth is None): smooth = 0
1285 if (smooth is None): smooth = 0
1286 elif (self.smooth < 3): smooth = 0
1286 elif (self.smooth < 3): smooth = 0
1287
1287
1288 if (type1 is None): type1 = 0
1288 if (type1 is None): type1 = 0
1289 if (fwindow is None): fwindow = numpy.zeros(oldfreq.size) + 1
1289 if (fwindow is None): fwindow = numpy.zeros(oldfreq.size) + 1
1290 if (snrth is None): snrth = -3
1290 if (snrth is None): snrth = -3
1291 if (dc is None): dc = 0
1291 if (dc is None): dc = 0
1292 if (aliasing is None): aliasing = 0
1292 if (aliasing is None): aliasing = 0
1293 if (oldfd is None): oldfd = 0
1293 if (oldfd is None): oldfd = 0
1294 if (wwauto is None): wwauto = 0
1294 if (wwauto is None): wwauto = 0
1295
1295
1296 if (n0 < 1.e-20): n0 = 1.e-20
1296 if (n0 < 1.e-20): n0 = 1.e-20
1297
1297
1298 freq = oldfreq
1298 freq = oldfreq
1299 vec_power = numpy.zeros(oldspec.shape[1])
1299 vec_power = numpy.zeros(oldspec.shape[1])
1300 vec_fd = numpy.zeros(oldspec.shape[1])
1300 vec_fd = numpy.zeros(oldspec.shape[1])
1301 vec_w = numpy.zeros(oldspec.shape[1])
1301 vec_w = numpy.zeros(oldspec.shape[1])
1302 vec_snr = numpy.zeros(oldspec.shape[1])
1302 vec_snr = numpy.zeros(oldspec.shape[1])
1303
1303
1304 # oldspec = numpy.ma.masked_invalid(oldspec)
1304 # oldspec = numpy.ma.masked_invalid(oldspec)
1305 for ind in range(oldspec.shape[1]):
1305 for ind in range(oldspec.shape[1]):
1306
1306
1307 spec = oldspec[:,ind]
1307 spec = oldspec[:,ind]
1308 aux = spec*fwindow
1308 aux = spec*fwindow
1309 max_spec = aux.max()
1309 max_spec = aux.max()
1310 m = aux.tolist().index(max_spec)
1310 m = aux.tolist().index(max_spec)
1311
1311
1312 # Smooth
1312 # Smooth
1313 if (smooth == 0):
1313 if (smooth == 0):
1314 spec2 = spec
1314 spec2 = spec
1315 else:
1315 else:
1316 spec2 = scipy.ndimage.filters.uniform_filter1d(spec,size=smooth)
1316 spec2 = scipy.ndimage.filters.uniform_filter1d(spec,size=smooth)
1317
1317
1318 # Moments Estimation
1318 # Moments Estimation
1319 bb = spec2[numpy.arange(m,spec2.size)]
1319 bb = spec2[numpy.arange(m,spec2.size)]
1320 bb = (bb<n0).nonzero()
1320 bb = (bb<n0).nonzero()
1321 bb = bb[0]
1321 bb = bb[0]
1322
1322
1323 ss = spec2[numpy.arange(0,m + 1)]
1323 ss = spec2[numpy.arange(0,m + 1)]
1324 ss = (ss<n0).nonzero()
1324 ss = (ss<n0).nonzero()
1325 ss = ss[0]
1325 ss = ss[0]
1326
1326
1327 if (bb.size == 0):
1327 if (bb.size == 0):
1328 bb0 = spec.size - 1 - m
1328 bb0 = spec.size - 1 - m
1329 else:
1329 else:
1330 bb0 = bb[0] - 1
1330 bb0 = bb[0] - 1
1331 if (bb0 < 0):
1331 if (bb0 < 0):
1332 bb0 = 0
1332 bb0 = 0
1333
1333
1334 if (ss.size == 0):
1334 if (ss.size == 0):
1335 ss1 = 1
1335 ss1 = 1
1336 else:
1336 else:
1337 ss1 = max(ss) + 1
1337 ss1 = max(ss) + 1
1338
1338
1339 if (ss1 > m):
1339 if (ss1 > m):
1340 ss1 = m
1340 ss1 = m
1341
1341
1342 #valid = numpy.arange(int(m + bb0 - ss1 + 1)) + ss1
1342 #valid = numpy.arange(int(m + bb0 - ss1 + 1)) + ss1
1343 valid = numpy.arange(1,oldspec.shape[0])# valid perfil completo igual pulsepair
1343 valid = numpy.arange(1,oldspec.shape[0])# valid perfil completo igual pulsepair
1344 signal_power = ((spec2[valid] - n0) * fwindow[valid]).mean() # D. Scipión added with correct definition
1344 signal_power = ((spec2[valid] - n0) * fwindow[valid]).mean() # D. Scipión added with correct definition
1345 total_power = (spec2[valid] * fwindow[valid]).mean() # D. Scipión added with correct definition
1345 total_power = (spec2[valid] * fwindow[valid]).mean() # D. Scipión added with correct definition
1346 power = ((spec2[valid] - n0) * fwindow[valid]).sum()
1346 power = ((spec2[valid] - n0) * fwindow[valid]).sum()
1347 fd = ((spec2[valid]- n0)*freq[valid] * fwindow[valid]).sum() / power
1347 fd = ((spec2[valid]- n0)*freq[valid] * fwindow[valid]).sum() / power
1348 w = numpy.sqrt(((spec2[valid] - n0)*fwindow[valid]*(freq[valid]- fd)**2).sum() / power)
1348 w = numpy.sqrt(((spec2[valid] - n0)*fwindow[valid]*(freq[valid]- fd)**2).sum() / power)
1349 snr = (spec2.mean()-n0)/n0
1349 snr = (spec2.mean()-n0)/n0
1350 if (snr < 1.e-20) :
1350 if (snr < 1.e-20) :
1351 snr = 1.e-20
1351 snr = 1.e-20
1352
1352
1353 # vec_power[ind] = power #D. Scipión replaced with the line below
1353 # vec_power[ind] = power #D. Scipión replaced with the line below
1354 if wradar ==False:
1354 if wradar ==False:
1355 vec_power[ind] = total_power
1355 vec_power[ind] = total_power
1356 else:
1356 else:
1357 vec_power[ind] = signal_power
1357 vec_power[ind] = signal_power
1358
1358
1359 vec_fd[ind] = fd
1359 vec_fd[ind] = fd
1360 vec_w[ind] = w
1360 vec_w[ind] = w
1361 vec_snr[ind] = snr
1361 vec_snr[ind] = snr
1362 return numpy.vstack((vec_snr, vec_power, vec_fd, vec_w))
1362 return numpy.vstack((vec_snr, vec_power, vec_fd, vec_w))
1363
1363
1364 #------------------ Get SA Parameters --------------------------
1364 #------------------ Get SA Parameters --------------------------
1365
1365
1366 def GetSAParameters(self):
1366 def GetSAParameters(self):
1367 #SA en frecuencia
1367 #SA en frecuencia
1368 pairslist = self.dataOut.groupList
1368 pairslist = self.dataOut.groupList
1369 num_pairs = len(pairslist)
1369 num_pairs = len(pairslist)
1370
1370
1371 vel = self.dataOut.abscissaList
1371 vel = self.dataOut.abscissaList
1372 spectra = self.dataOut.data_pre
1372 spectra = self.dataOut.data_pre
1373 cspectra = self.dataIn.data_cspc
1373 cspectra = self.dataIn.data_cspc
1374 delta_v = vel[1] - vel[0]
1374 delta_v = vel[1] - vel[0]
1375
1375
1376 #Calculating the power spectrum
1376 #Calculating the power spectrum
1377 spc_pow = numpy.sum(spectra, 3)*delta_v
1377 spc_pow = numpy.sum(spectra, 3)*delta_v
1378 #Normalizing Spectra
1378 #Normalizing Spectra
1379 norm_spectra = spectra/spc_pow
1379 norm_spectra = spectra/spc_pow
1380 #Calculating the norm_spectra at peak
1380 #Calculating the norm_spectra at peak
1381 max_spectra = numpy.max(norm_spectra, 3)
1381 max_spectra = numpy.max(norm_spectra, 3)
1382
1382
1383 #Normalizing Cross Spectra
1383 #Normalizing Cross Spectra
1384 norm_cspectra = numpy.zeros(cspectra.shape)
1384 norm_cspectra = numpy.zeros(cspectra.shape)
1385
1385
1386 for i in range(num_chan):
1386 for i in range(num_chan):
1387 norm_cspectra[i,:,:] = cspectra[i,:,:]/numpy.sqrt(spc_pow[pairslist[i][0],:]*spc_pow[pairslist[i][1],:])
1387 norm_cspectra[i,:,:] = cspectra[i,:,:]/numpy.sqrt(spc_pow[pairslist[i][0],:]*spc_pow[pairslist[i][1],:])
1388
1388
1389 max_cspectra = numpy.max(norm_cspectra,2)
1389 max_cspectra = numpy.max(norm_cspectra,2)
1390 max_cspectra_index = numpy.argmax(norm_cspectra, 2)
1390 max_cspectra_index = numpy.argmax(norm_cspectra, 2)
1391
1391
1392 for i in range(num_pairs):
1392 for i in range(num_pairs):
1393 cspc_par[i,:,:] = __calculateMoments(norm_cspectra)
1393 cspc_par[i,:,:] = __calculateMoments(norm_cspectra)
1394 #------------------- Get Lags ----------------------------------
1394 #------------------- Get Lags ----------------------------------
1395
1395
1396 class SALags(Operation):
1396 class SALags(Operation):
1397 '''
1397 '''
1398 Function GetMoments()
1398 Function GetMoments()
1399
1399
1400 Input:
1400 Input:
1401 self.dataOut.data_pre
1401 self.dataOut.data_pre
1402 self.dataOut.abscissaList
1402 self.dataOut.abscissaList
1403 self.dataOut.noise
1403 self.dataOut.noise
1404 self.dataOut.normFactor
1404 self.dataOut.normFactor
1405 self.dataOut.data_snr
1405 self.dataOut.data_snr
1406 self.dataOut.groupList
1406 self.dataOut.groupList
1407 self.dataOut.nChannels
1407 self.dataOut.nChannels
1408
1408
1409 Affected:
1409 Affected:
1410 self.dataOut.data_param
1410 self.dataOut.data_param
1411
1411
1412 '''
1412 '''
1413 def run(self, dataOut):
1413 def run(self, dataOut):
1414 data_acf = dataOut.data_pre[0]
1414 data_acf = dataOut.data_pre[0]
1415 data_ccf = dataOut.data_pre[1]
1415 data_ccf = dataOut.data_pre[1]
1416 normFactor_acf = dataOut.normFactor[0]
1416 normFactor_acf = dataOut.normFactor[0]
1417 normFactor_ccf = dataOut.normFactor[1]
1417 normFactor_ccf = dataOut.normFactor[1]
1418 pairs_acf = dataOut.groupList[0]
1418 pairs_acf = dataOut.groupList[0]
1419 pairs_ccf = dataOut.groupList[1]
1419 pairs_ccf = dataOut.groupList[1]
1420
1420
1421 nHeights = dataOut.nHeights
1421 nHeights = dataOut.nHeights
1422 absc = dataOut.abscissaList
1422 absc = dataOut.abscissaList
1423 noise = dataOut.noise
1423 noise = dataOut.noise
1424 SNR = dataOut.data_snr
1424 SNR = dataOut.data_snr
1425 nChannels = dataOut.nChannels
1425 nChannels = dataOut.nChannels
1426 # pairsList = dataOut.groupList
1426 # pairsList = dataOut.groupList
1427 # pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairsList, nChannels)
1427 # pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairsList, nChannels)
1428
1428
1429 for l in range(len(pairs_acf)):
1429 for l in range(len(pairs_acf)):
1430 data_acf[l,:,:] = data_acf[l,:,:]/normFactor_acf[l,:]
1430 data_acf[l,:,:] = data_acf[l,:,:]/normFactor_acf[l,:]
1431
1431
1432 for l in range(len(pairs_ccf)):
1432 for l in range(len(pairs_ccf)):
1433 data_ccf[l,:,:] = data_ccf[l,:,:]/normFactor_ccf[l,:]
1433 data_ccf[l,:,:] = data_ccf[l,:,:]/normFactor_ccf[l,:]
1434
1434
1435 dataOut.data_param = numpy.zeros((len(pairs_ccf)*2 + 1, nHeights))
1435 dataOut.data_param = numpy.zeros((len(pairs_ccf)*2 + 1, nHeights))
1436 dataOut.data_param[:-1,:] = self.__calculateTaus(data_acf, data_ccf, absc)
1436 dataOut.data_param[:-1,:] = self.__calculateTaus(data_acf, data_ccf, absc)
1437 dataOut.data_param[-1,:] = self.__calculateLag1Phase(data_acf, absc)
1437 dataOut.data_param[-1,:] = self.__calculateLag1Phase(data_acf, absc)
1438 return
1438 return
1439
1439
1440 # def __getPairsAutoCorr(self, pairsList, nChannels):
1440 # def __getPairsAutoCorr(self, pairsList, nChannels):
1441 #
1441 #
1442 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
1442 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
1443 #
1443 #
1444 # for l in range(len(pairsList)):
1444 # for l in range(len(pairsList)):
1445 # firstChannel = pairsList[l][0]
1445 # firstChannel = pairsList[l][0]
1446 # secondChannel = pairsList[l][1]
1446 # secondChannel = pairsList[l][1]
1447 #
1447 #
1448 # #Obteniendo pares de Autocorrelacion
1448 # #Obteniendo pares de Autocorrelacion
1449 # if firstChannel == secondChannel:
1449 # if firstChannel == secondChannel:
1450 # pairsAutoCorr[firstChannel] = int(l)
1450 # pairsAutoCorr[firstChannel] = int(l)
1451 #
1451 #
1452 # pairsAutoCorr = pairsAutoCorr.astype(int)
1452 # pairsAutoCorr = pairsAutoCorr.astype(int)
1453 #
1453 #
1454 # pairsCrossCorr = range(len(pairsList))
1454 # pairsCrossCorr = range(len(pairsList))
1455 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
1455 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
1456 #
1456 #
1457 # return pairsAutoCorr, pairsCrossCorr
1457 # return pairsAutoCorr, pairsCrossCorr
1458
1458
1459 def __calculateTaus(self, data_acf, data_ccf, lagRange):
1459 def __calculateTaus(self, data_acf, data_ccf, lagRange):
1460
1460
1461 lag0 = data_acf.shape[1]/2
1461 lag0 = data_acf.shape[1]/2
1462 #Funcion de Autocorrelacion
1462 #Funcion de Autocorrelacion
1463 mean_acf = stats.nanmean(data_acf, axis = 0)
1463 mean_acf = stats.nanmean(data_acf, axis = 0)
1464
1464
1465 #Obtencion Indice de TauCross
1465 #Obtencion Indice de TauCross
1466 ind_ccf = data_ccf.argmax(axis = 1)
1466 ind_ccf = data_ccf.argmax(axis = 1)
1467 #Obtencion Indice de TauAuto
1467 #Obtencion Indice de TauAuto
1468 ind_acf = numpy.zeros(ind_ccf.shape,dtype = 'int')
1468 ind_acf = numpy.zeros(ind_ccf.shape,dtype = 'int')
1469 ccf_lag0 = data_ccf[:,lag0,:]
1469 ccf_lag0 = data_ccf[:,lag0,:]
1470
1470
1471 for i in range(ccf_lag0.shape[0]):
1471 for i in range(ccf_lag0.shape[0]):
1472 ind_acf[i,:] = numpy.abs(mean_acf - ccf_lag0[i,:]).argmin(axis = 0)
1472 ind_acf[i,:] = numpy.abs(mean_acf - ccf_lag0[i,:]).argmin(axis = 0)
1473
1473
1474 #Obtencion de TauCross y TauAuto
1474 #Obtencion de TauCross y TauAuto
1475 tau_ccf = lagRange[ind_ccf]
1475 tau_ccf = lagRange[ind_ccf]
1476 tau_acf = lagRange[ind_acf]
1476 tau_acf = lagRange[ind_acf]
1477
1477
1478 Nan1, Nan2 = numpy.where(tau_ccf == lagRange[0])
1478 Nan1, Nan2 = numpy.where(tau_ccf == lagRange[0])
1479
1479
1480 tau_ccf[Nan1,Nan2] = numpy.nan
1480 tau_ccf[Nan1,Nan2] = numpy.nan
1481 tau_acf[Nan1,Nan2] = numpy.nan
1481 tau_acf[Nan1,Nan2] = numpy.nan
1482 tau = numpy.vstack((tau_ccf,tau_acf))
1482 tau = numpy.vstack((tau_ccf,tau_acf))
1483
1483
1484 return tau
1484 return tau
1485
1485
1486 def __calculateLag1Phase(self, data, lagTRange):
1486 def __calculateLag1Phase(self, data, lagTRange):
1487 data1 = stats.nanmean(data, axis = 0)
1487 data1 = stats.nanmean(data, axis = 0)
1488 lag1 = numpy.where(lagTRange == 0)[0][0] + 1
1488 lag1 = numpy.where(lagTRange == 0)[0][0] + 1
1489
1489
1490 phase = numpy.angle(data1[lag1,:])
1490 phase = numpy.angle(data1[lag1,:])
1491
1491
1492 return phase
1492 return phase
1493
1493
1494 class SpectralFitting(Operation):
1494 class SpectralFitting(Operation):
1495 '''
1495 '''
1496 Function GetMoments()
1496 Function GetMoments()
1497
1497
1498 Input:
1498 Input:
1499 Output:
1499 Output:
1500 Variables modified:
1500 Variables modified:
1501 '''
1501 '''
1502
1502
1503 def run(self, dataOut, getSNR = True, path=None, file=None, groupList=None):
1503 def run(self, dataOut, getSNR = True, path=None, file=None, groupList=None):
1504
1504
1505
1505
1506 if path != None:
1506 if path != None:
1507 sys.path.append(path)
1507 sys.path.append(path)
1508 self.dataOut.library = importlib.import_module(file)
1508 self.dataOut.library = importlib.import_module(file)
1509
1509
1510 #To be inserted as a parameter
1510 #To be inserted as a parameter
1511 groupArray = numpy.array(groupList)
1511 groupArray = numpy.array(groupList)
1512 # groupArray = numpy.array([[0,1],[2,3]])
1512 # groupArray = numpy.array([[0,1],[2,3]])
1513 self.dataOut.groupList = groupArray
1513 self.dataOut.groupList = groupArray
1514
1514
1515 nGroups = groupArray.shape[0]
1515 nGroups = groupArray.shape[0]
1516 nChannels = self.dataIn.nChannels
1516 nChannels = self.dataIn.nChannels
1517 nHeights=self.dataIn.heightList.size
1517 nHeights=self.dataIn.heightList.size
1518
1518
1519 #Parameters Array
1519 #Parameters Array
1520 self.dataOut.data_param = None
1520 self.dataOut.data_param = None
1521
1521
1522 #Set constants
1522 #Set constants
1523 constants = self.dataOut.library.setConstants(self.dataIn)
1523 constants = self.dataOut.library.setConstants(self.dataIn)
1524 self.dataOut.constants = constants
1524 self.dataOut.constants = constants
1525 M = self.dataIn.normFactor
1525 M = self.dataIn.normFactor
1526 N = self.dataIn.nFFTPoints
1526 N = self.dataIn.nFFTPoints
1527 ippSeconds = self.dataIn.ippSeconds
1527 ippSeconds = self.dataIn.ippSeconds
1528 K = self.dataIn.nIncohInt
1528 K = self.dataIn.nIncohInt
1529 pairsArray = numpy.array(self.dataIn.pairsList)
1529 pairsArray = numpy.array(self.dataIn.pairsList)
1530
1530
1531 #List of possible combinations
1531 #List of possible combinations
1532 listComb = itertools.combinations(numpy.arange(groupArray.shape[1]),2)
1532 listComb = itertools.combinations(numpy.arange(groupArray.shape[1]),2)
1533 indCross = numpy.zeros(len(list(listComb)), dtype = 'int')
1533 indCross = numpy.zeros(len(list(listComb)), dtype = 'int')
1534
1534
1535 if getSNR:
1535 if getSNR:
1536 listChannels = groupArray.reshape((groupArray.size))
1536 listChannels = groupArray.reshape((groupArray.size))
1537 listChannels.sort()
1537 listChannels.sort()
1538 noise = self.dataIn.getNoise()
1538 noise = self.dataIn.getNoise()
1539 self.dataOut.data_snr = self.__getSNR(self.dataIn.data_spc[listChannels,:,:], noise[listChannels])
1539 self.dataOut.data_snr = self.__getSNR(self.dataIn.data_spc[listChannels,:,:], noise[listChannels])
1540
1540
1541 for i in range(nGroups):
1541 for i in range(nGroups):
1542 coord = groupArray[i,:]
1542 coord = groupArray[i,:]
1543
1543
1544 #Input data array
1544 #Input data array
1545 data = self.dataIn.data_spc[coord,:,:]/(M*N)
1545 data = self.dataIn.data_spc[coord,:,:]/(M*N)
1546 data = data.reshape((data.shape[0]*data.shape[1],data.shape[2]))
1546 data = data.reshape((data.shape[0]*data.shape[1],data.shape[2]))
1547
1547
1548 #Cross Spectra data array for Covariance Matrixes
1548 #Cross Spectra data array for Covariance Matrixes
1549 ind = 0
1549 ind = 0
1550 for pairs in listComb:
1550 for pairs in listComb:
1551 pairsSel = numpy.array([coord[x],coord[y]])
1551 pairsSel = numpy.array([coord[x],coord[y]])
1552 indCross[ind] = int(numpy.where(numpy.all(pairsArray == pairsSel, axis = 1))[0][0])
1552 indCross[ind] = int(numpy.where(numpy.all(pairsArray == pairsSel, axis = 1))[0][0])
1553 ind += 1
1553 ind += 1
1554 dataCross = self.dataIn.data_cspc[indCross,:,:]/(M*N)
1554 dataCross = self.dataIn.data_cspc[indCross,:,:]/(M*N)
1555 dataCross = dataCross**2/K
1555 dataCross = dataCross**2/K
1556
1556
1557 for h in range(nHeights):
1557 for h in range(nHeights):
1558
1558
1559 #Input
1559 #Input
1560 d = data[:,h]
1560 d = data[:,h]
1561
1561
1562 #Covariance Matrix
1562 #Covariance Matrix
1563 D = numpy.diag(d**2/K)
1563 D = numpy.diag(d**2/K)
1564 ind = 0
1564 ind = 0
1565 for pairs in listComb:
1565 for pairs in listComb:
1566 #Coordinates in Covariance Matrix
1566 #Coordinates in Covariance Matrix
1567 x = pairs[0]
1567 x = pairs[0]
1568 y = pairs[1]
1568 y = pairs[1]
1569 #Channel Index
1569 #Channel Index
1570 S12 = dataCross[ind,:,h]
1570 S12 = dataCross[ind,:,h]
1571 D12 = numpy.diag(S12)
1571 D12 = numpy.diag(S12)
1572 #Completing Covariance Matrix with Cross Spectras
1572 #Completing Covariance Matrix with Cross Spectras
1573 D[x*N:(x+1)*N,y*N:(y+1)*N] = D12
1573 D[x*N:(x+1)*N,y*N:(y+1)*N] = D12
1574 D[y*N:(y+1)*N,x*N:(x+1)*N] = D12
1574 D[y*N:(y+1)*N,x*N:(x+1)*N] = D12
1575 ind += 1
1575 ind += 1
1576 Dinv=numpy.linalg.inv(D)
1576 Dinv=numpy.linalg.inv(D)
1577 L=numpy.linalg.cholesky(Dinv)
1577 L=numpy.linalg.cholesky(Dinv)
1578 LT=L.T
1578 LT=L.T
1579
1579
1580 dp = numpy.dot(LT,d)
1580 dp = numpy.dot(LT,d)
1581
1581
1582 #Initial values
1582 #Initial values
1583 data_spc = self.dataIn.data_spc[coord,:,h]
1583 data_spc = self.dataIn.data_spc[coord,:,h]
1584
1584
1585 if (h>0)and(error1[3]<5):
1585 if (h>0)and(error1[3]<5):
1586 p0 = self.dataOut.data_param[i,:,h-1]
1586 p0 = self.dataOut.data_param[i,:,h-1]
1587 else:
1587 else:
1588 p0 = numpy.array(self.dataOut.library.initialValuesFunction(data_spc, constants, i))
1588 p0 = numpy.array(self.dataOut.library.initialValuesFunction(data_spc, constants, i))
1589
1589
1590 try:
1590 try:
1591 #Least Squares
1591 #Least Squares
1592 minp,covp,infodict,mesg,ier = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants),full_output=True)
1592 minp,covp,infodict,mesg,ier = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants),full_output=True)
1593 # minp,covp = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants))
1593 # minp,covp = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants))
1594 #Chi square error
1594 #Chi square error
1595 error0 = numpy.sum(infodict['fvec']**2)/(2*N)
1595 error0 = numpy.sum(infodict['fvec']**2)/(2*N)
1596 #Error with Jacobian
1596 #Error with Jacobian
1597 error1 = self.dataOut.library.errorFunction(minp,constants,LT)
1597 error1 = self.dataOut.library.errorFunction(minp,constants,LT)
1598 except:
1598 except:
1599 minp = p0*numpy.nan
1599 minp = p0*numpy.nan
1600 error0 = numpy.nan
1600 error0 = numpy.nan
1601 error1 = p0*numpy.nan
1601 error1 = p0*numpy.nan
1602
1602
1603 #Save
1603 #Save
1604 if self.dataOut.data_param is None:
1604 if self.dataOut.data_param is None:
1605 self.dataOut.data_param = numpy.zeros((nGroups, p0.size, nHeights))*numpy.nan
1605 self.dataOut.data_param = numpy.zeros((nGroups, p0.size, nHeights))*numpy.nan
1606 self.dataOut.data_error = numpy.zeros((nGroups, p0.size + 1, nHeights))*numpy.nan
1606 self.dataOut.data_error = numpy.zeros((nGroups, p0.size + 1, nHeights))*numpy.nan
1607
1607
1608 self.dataOut.data_error[i,:,h] = numpy.hstack((error0,error1))
1608 self.dataOut.data_error[i,:,h] = numpy.hstack((error0,error1))
1609 self.dataOut.data_param[i,:,h] = minp
1609 self.dataOut.data_param[i,:,h] = minp
1610 return
1610 return
1611
1611
1612 def __residFunction(self, p, dp, LT, constants):
1612 def __residFunction(self, p, dp, LT, constants):
1613
1613
1614 fm = self.dataOut.library.modelFunction(p, constants)
1614 fm = self.dataOut.library.modelFunction(p, constants)
1615 fmp=numpy.dot(LT,fm)
1615 fmp=numpy.dot(LT,fm)
1616
1616
1617 return dp-fmp
1617 return dp-fmp
1618
1618
1619 def __getSNR(self, z, noise):
1619 def __getSNR(self, z, noise):
1620
1620
1621 avg = numpy.average(z, axis=1)
1621 avg = numpy.average(z, axis=1)
1622 SNR = (avg.T-noise)/noise
1622 SNR = (avg.T-noise)/noise
1623 SNR = SNR.T
1623 SNR = SNR.T
1624 return SNR
1624 return SNR
1625
1625
1626 def __chisq(p,chindex,hindex):
1626 def __chisq(p,chindex,hindex):
1627 #similar to Resid but calculates CHI**2
1627 #similar to Resid but calculates CHI**2
1628 [LT,d,fm]=setupLTdfm(p,chindex,hindex)
1628 [LT,d,fm]=setupLTdfm(p,chindex,hindex)
1629 dp=numpy.dot(LT,d)
1629 dp=numpy.dot(LT,d)
1630 fmp=numpy.dot(LT,fm)
1630 fmp=numpy.dot(LT,fm)
1631 chisq=numpy.dot((dp-fmp).T,(dp-fmp))
1631 chisq=numpy.dot((dp-fmp).T,(dp-fmp))
1632 return chisq
1632 return chisq
1633
1633
1634 class WindProfiler(Operation):
1634 class WindProfiler(Operation):
1635
1635
1636 __isConfig = False
1636 __isConfig = False
1637
1637
1638 __initime = None
1638 __initime = None
1639 __lastdatatime = None
1639 __lastdatatime = None
1640 __integrationtime = None
1640 __integrationtime = None
1641
1641
1642 __buffer = None
1642 __buffer = None
1643
1643
1644 __dataReady = False
1644 __dataReady = False
1645
1645
1646 __firstdata = None
1646 __firstdata = None
1647
1647
1648 n = None
1648 n = None
1649
1649
1650 def __init__(self):
1650 def __init__(self):
1651 Operation.__init__(self)
1651 Operation.__init__(self)
1652
1652
1653 def __calculateCosDir(self, elev, azim):
1653 def __calculateCosDir(self, elev, azim):
1654 zen = (90 - elev)*numpy.pi/180
1654 zen = (90 - elev)*numpy.pi/180
1655 azim = azim*numpy.pi/180
1655 azim = azim*numpy.pi/180
1656 cosDirX = numpy.sqrt((1-numpy.cos(zen)**2)/((1+numpy.tan(azim)**2)))
1656 cosDirX = numpy.sqrt((1-numpy.cos(zen)**2)/((1+numpy.tan(azim)**2)))
1657 cosDirY = numpy.sqrt(1-numpy.cos(zen)**2-cosDirX**2)
1657 cosDirY = numpy.sqrt(1-numpy.cos(zen)**2-cosDirX**2)
1658
1658
1659 signX = numpy.sign(numpy.cos(azim))
1659 signX = numpy.sign(numpy.cos(azim))
1660 signY = numpy.sign(numpy.sin(azim))
1660 signY = numpy.sign(numpy.sin(azim))
1661
1661
1662 cosDirX = numpy.copysign(cosDirX, signX)
1662 cosDirX = numpy.copysign(cosDirX, signX)
1663 cosDirY = numpy.copysign(cosDirY, signY)
1663 cosDirY = numpy.copysign(cosDirY, signY)
1664 return cosDirX, cosDirY
1664 return cosDirX, cosDirY
1665
1665
1666 def __calculateAngles(self, theta_x, theta_y, azimuth):
1666 def __calculateAngles(self, theta_x, theta_y, azimuth):
1667
1667
1668 dir_cosw = numpy.sqrt(1-theta_x**2-theta_y**2)
1668 dir_cosw = numpy.sqrt(1-theta_x**2-theta_y**2)
1669 zenith_arr = numpy.arccos(dir_cosw)
1669 zenith_arr = numpy.arccos(dir_cosw)
1670 azimuth_arr = numpy.arctan2(theta_x,theta_y) + azimuth*math.pi/180
1670 azimuth_arr = numpy.arctan2(theta_x,theta_y) + azimuth*math.pi/180
1671
1671
1672 dir_cosu = numpy.sin(azimuth_arr)*numpy.sin(zenith_arr)
1672 dir_cosu = numpy.sin(azimuth_arr)*numpy.sin(zenith_arr)
1673 dir_cosv = numpy.cos(azimuth_arr)*numpy.sin(zenith_arr)
1673 dir_cosv = numpy.cos(azimuth_arr)*numpy.sin(zenith_arr)
1674
1674
1675 return azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw
1675 return azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw
1676
1676
1677 def __calculateMatA(self, dir_cosu, dir_cosv, dir_cosw, horOnly):
1677 def __calculateMatA(self, dir_cosu, dir_cosv, dir_cosw, horOnly):
1678
1678
1679 #
1679 #
1680 if horOnly:
1680 if horOnly:
1681 A = numpy.c_[dir_cosu,dir_cosv]
1681 A = numpy.c_[dir_cosu,dir_cosv]
1682 else:
1682 else:
1683 A = numpy.c_[dir_cosu,dir_cosv,dir_cosw]
1683 A = numpy.c_[dir_cosu,dir_cosv,dir_cosw]
1684 A = numpy.asmatrix(A)
1684 A = numpy.asmatrix(A)
1685 A1 = numpy.linalg.inv(A.transpose()*A)*A.transpose()
1685 A1 = numpy.linalg.inv(A.transpose()*A)*A.transpose()
1686
1686
1687 return A1
1687 return A1
1688
1688
1689 def __correctValues(self, heiRang, phi, velRadial, SNR):
1689 def __correctValues(self, heiRang, phi, velRadial, SNR):
1690 listPhi = phi.tolist()
1690 listPhi = phi.tolist()
1691 maxid = listPhi.index(max(listPhi))
1691 maxid = listPhi.index(max(listPhi))
1692 minid = listPhi.index(min(listPhi))
1692 minid = listPhi.index(min(listPhi))
1693
1693
1694 rango = list(range(len(phi)))
1694 rango = list(range(len(phi)))
1695 # rango = numpy.delete(rango,maxid)
1695 # rango = numpy.delete(rango,maxid)
1696
1696
1697 heiRang1 = heiRang*math.cos(phi[maxid])
1697 heiRang1 = heiRang*math.cos(phi[maxid])
1698 heiRangAux = heiRang*math.cos(phi[minid])
1698 heiRangAux = heiRang*math.cos(phi[minid])
1699 indOut = (heiRang1 < heiRangAux[0]).nonzero()
1699 indOut = (heiRang1 < heiRangAux[0]).nonzero()
1700 heiRang1 = numpy.delete(heiRang1,indOut)
1700 heiRang1 = numpy.delete(heiRang1,indOut)
1701
1701
1702 velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
1702 velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
1703 SNR1 = numpy.zeros([len(phi),len(heiRang1)])
1703 SNR1 = numpy.zeros([len(phi),len(heiRang1)])
1704
1704
1705 for i in rango:
1705 for i in rango:
1706 x = heiRang*math.cos(phi[i])
1706 x = heiRang*math.cos(phi[i])
1707 y1 = velRadial[i,:]
1707 y1 = velRadial[i,:]
1708 f1 = interpolate.interp1d(x,y1,kind = 'cubic')
1708 f1 = interpolate.interp1d(x,y1,kind = 'cubic')
1709
1709
1710 x1 = heiRang1
1710 x1 = heiRang1
1711 y11 = f1(x1)
1711 y11 = f1(x1)
1712
1712
1713 y2 = SNR[i,:]
1713 y2 = SNR[i,:]
1714 f2 = interpolate.interp1d(x,y2,kind = 'cubic')
1714 f2 = interpolate.interp1d(x,y2,kind = 'cubic')
1715 y21 = f2(x1)
1715 y21 = f2(x1)
1716
1716
1717 velRadial1[i,:] = y11
1717 velRadial1[i,:] = y11
1718 SNR1[i,:] = y21
1718 SNR1[i,:] = y21
1719
1719
1720 return heiRang1, velRadial1, SNR1
1720 return heiRang1, velRadial1, SNR1
1721
1721
1722 def __calculateVelUVW(self, A, velRadial):
1722 def __calculateVelUVW(self, A, velRadial):
1723
1723
1724 #Operacion Matricial
1724 #Operacion Matricial
1725 # velUVW = numpy.zeros((velRadial.shape[1],3))
1725 # velUVW = numpy.zeros((velRadial.shape[1],3))
1726 # for ind in range(velRadial.shape[1]):
1726 # for ind in range(velRadial.shape[1]):
1727 # velUVW[ind,:] = numpy.dot(A,velRadial[:,ind])
1727 # velUVW[ind,:] = numpy.dot(A,velRadial[:,ind])
1728 # velUVW = velUVW.transpose()
1728 # velUVW = velUVW.transpose()
1729 velUVW = numpy.zeros((A.shape[0],velRadial.shape[1]))
1729 velUVW = numpy.zeros((A.shape[0],velRadial.shape[1]))
1730 velUVW[:,:] = numpy.dot(A,velRadial)
1730 velUVW[:,:] = numpy.dot(A,velRadial)
1731
1731
1732
1732
1733 return velUVW
1733 return velUVW
1734
1734
1735 # def techniqueDBS(self, velRadial0, dirCosx, disrCosy, azimuth, correct, horizontalOnly, heiRang, SNR0):
1735 # def techniqueDBS(self, velRadial0, dirCosx, disrCosy, azimuth, correct, horizontalOnly, heiRang, SNR0):
1736
1736
1737 def techniqueDBS(self, kwargs):
1737 def techniqueDBS(self, kwargs):
1738 """
1738 """
1739 Function that implements Doppler Beam Swinging (DBS) technique.
1739 Function that implements Doppler Beam Swinging (DBS) technique.
1740
1740
1741 Input: Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
1741 Input: Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
1742 Direction correction (if necessary), Ranges and SNR
1742 Direction correction (if necessary), Ranges and SNR
1743
1743
1744 Output: Winds estimation (Zonal, Meridional and Vertical)
1744 Output: Winds estimation (Zonal, Meridional and Vertical)
1745
1745
1746 Parameters affected: Winds, height range, SNR
1746 Parameters affected: Winds, height range, SNR
1747 """
1747 """
1748 velRadial0 = kwargs['velRadial']
1748 velRadial0 = kwargs['velRadial']
1749 heiRang = kwargs['heightList']
1749 heiRang = kwargs['heightList']
1750 SNR0 = kwargs['SNR']
1750 SNR0 = kwargs['SNR']
1751
1751
1752 if 'dirCosx' in kwargs and 'dirCosy' in kwargs:
1752 if 'dirCosx' in kwargs and 'dirCosy' in kwargs:
1753 theta_x = numpy.array(kwargs['dirCosx'])
1753 theta_x = numpy.array(kwargs['dirCosx'])
1754 theta_y = numpy.array(kwargs['dirCosy'])
1754 theta_y = numpy.array(kwargs['dirCosy'])
1755 else:
1755 else:
1756 elev = numpy.array(kwargs['elevation'])
1756 elev = numpy.array(kwargs['elevation'])
1757 azim = numpy.array(kwargs['azimuth'])
1757 azim = numpy.array(kwargs['azimuth'])
1758 theta_x, theta_y = self.__calculateCosDir(elev, azim)
1758 theta_x, theta_y = self.__calculateCosDir(elev, azim)
1759 azimuth = kwargs['correctAzimuth']
1759 azimuth = kwargs['correctAzimuth']
1760 if 'horizontalOnly' in kwargs:
1760 if 'horizontalOnly' in kwargs:
1761 horizontalOnly = kwargs['horizontalOnly']
1761 horizontalOnly = kwargs['horizontalOnly']
1762 else: horizontalOnly = False
1762 else: horizontalOnly = False
1763 if 'correctFactor' in kwargs:
1763 if 'correctFactor' in kwargs:
1764 correctFactor = kwargs['correctFactor']
1764 correctFactor = kwargs['correctFactor']
1765 else: correctFactor = 1
1765 else: correctFactor = 1
1766 if 'channelList' in kwargs:
1766 if 'channelList' in kwargs:
1767 channelList = kwargs['channelList']
1767 channelList = kwargs['channelList']
1768 if len(channelList) == 2:
1768 if len(channelList) == 2:
1769 horizontalOnly = True
1769 horizontalOnly = True
1770 arrayChannel = numpy.array(channelList)
1770 arrayChannel = numpy.array(channelList)
1771 param = param[arrayChannel,:,:]
1771 param = param[arrayChannel,:,:]
1772 theta_x = theta_x[arrayChannel]
1772 theta_x = theta_x[arrayChannel]
1773 theta_y = theta_y[arrayChannel]
1773 theta_y = theta_y[arrayChannel]
1774
1774
1775 azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw = self.__calculateAngles(theta_x, theta_y, azimuth)
1775 azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw = self.__calculateAngles(theta_x, theta_y, azimuth)
1776 heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, zenith_arr, correctFactor*velRadial0, SNR0)
1776 heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, zenith_arr, correctFactor*velRadial0, SNR0)
1777 A = self.__calculateMatA(dir_cosu, dir_cosv, dir_cosw, horizontalOnly)
1777 A = self.__calculateMatA(dir_cosu, dir_cosv, dir_cosw, horizontalOnly)
1778
1778
1779 #Calculo de Componentes de la velocidad con DBS
1779 #Calculo de Componentes de la velocidad con DBS
1780 winds = self.__calculateVelUVW(A,velRadial1)
1780 winds = self.__calculateVelUVW(A,velRadial1)
1781
1781
1782 return winds, heiRang1, SNR1
1782 return winds, heiRang1, SNR1
1783
1783
1784 def __calculateDistance(self, posx, posy, pairs_ccf, azimuth = None):
1784 def __calculateDistance(self, posx, posy, pairs_ccf, azimuth = None):
1785
1785
1786 nPairs = len(pairs_ccf)
1786 nPairs = len(pairs_ccf)
1787 posx = numpy.asarray(posx)
1787 posx = numpy.asarray(posx)
1788 posy = numpy.asarray(posy)
1788 posy = numpy.asarray(posy)
1789
1789
1790 #Rotacion Inversa para alinear con el azimuth
1790 #Rotacion Inversa para alinear con el azimuth
1791 if azimuth!= None:
1791 if azimuth!= None:
1792 azimuth = azimuth*math.pi/180
1792 azimuth = azimuth*math.pi/180
1793 posx1 = posx*math.cos(azimuth) + posy*math.sin(azimuth)
1793 posx1 = posx*math.cos(azimuth) + posy*math.sin(azimuth)
1794 posy1 = -posx*math.sin(azimuth) + posy*math.cos(azimuth)
1794 posy1 = -posx*math.sin(azimuth) + posy*math.cos(azimuth)
1795 else:
1795 else:
1796 posx1 = posx
1796 posx1 = posx
1797 posy1 = posy
1797 posy1 = posy
1798
1798
1799 #Calculo de Distancias
1799 #Calculo de Distancias
1800 distx = numpy.zeros(nPairs)
1800 distx = numpy.zeros(nPairs)
1801 disty = numpy.zeros(nPairs)
1801 disty = numpy.zeros(nPairs)
1802 dist = numpy.zeros(nPairs)
1802 dist = numpy.zeros(nPairs)
1803 ang = numpy.zeros(nPairs)
1803 ang = numpy.zeros(nPairs)
1804
1804
1805 for i in range(nPairs):
1805 for i in range(nPairs):
1806 distx[i] = posx1[pairs_ccf[i][1]] - posx1[pairs_ccf[i][0]]
1806 distx[i] = posx1[pairs_ccf[i][1]] - posx1[pairs_ccf[i][0]]
1807 disty[i] = posy1[pairs_ccf[i][1]] - posy1[pairs_ccf[i][0]]
1807 disty[i] = posy1[pairs_ccf[i][1]] - posy1[pairs_ccf[i][0]]
1808 dist[i] = numpy.sqrt(distx[i]**2 + disty[i]**2)
1808 dist[i] = numpy.sqrt(distx[i]**2 + disty[i]**2)
1809 ang[i] = numpy.arctan2(disty[i],distx[i])
1809 ang[i] = numpy.arctan2(disty[i],distx[i])
1810
1810
1811 return distx, disty, dist, ang
1811 return distx, disty, dist, ang
1812 #Calculo de Matrices
1812 #Calculo de Matrices
1813 # nPairs = len(pairs)
1813 # nPairs = len(pairs)
1814 # ang1 = numpy.zeros((nPairs, 2, 1))
1814 # ang1 = numpy.zeros((nPairs, 2, 1))
1815 # dist1 = numpy.zeros((nPairs, 2, 1))
1815 # dist1 = numpy.zeros((nPairs, 2, 1))
1816 #
1816 #
1817 # for j in range(nPairs):
1817 # for j in range(nPairs):
1818 # dist1[j,0,0] = dist[pairs[j][0]]
1818 # dist1[j,0,0] = dist[pairs[j][0]]
1819 # dist1[j,1,0] = dist[pairs[j][1]]
1819 # dist1[j,1,0] = dist[pairs[j][1]]
1820 # ang1[j,0,0] = ang[pairs[j][0]]
1820 # ang1[j,0,0] = ang[pairs[j][0]]
1821 # ang1[j,1,0] = ang[pairs[j][1]]
1821 # ang1[j,1,0] = ang[pairs[j][1]]
1822 #
1822 #
1823 # return distx,disty, dist1,ang1
1823 # return distx,disty, dist1,ang1
1824
1824
1825
1825
1826 def __calculateVelVer(self, phase, lagTRange, _lambda):
1826 def __calculateVelVer(self, phase, lagTRange, _lambda):
1827
1827
1828 Ts = lagTRange[1] - lagTRange[0]
1828 Ts = lagTRange[1] - lagTRange[0]
1829 velW = -_lambda*phase/(4*math.pi*Ts)
1829 velW = -_lambda*phase/(4*math.pi*Ts)
1830
1830
1831 return velW
1831 return velW
1832
1832
1833 def __calculateVelHorDir(self, dist, tau1, tau2, ang):
1833 def __calculateVelHorDir(self, dist, tau1, tau2, ang):
1834 nPairs = tau1.shape[0]
1834 nPairs = tau1.shape[0]
1835 nHeights = tau1.shape[1]
1835 nHeights = tau1.shape[1]
1836 vel = numpy.zeros((nPairs,3,nHeights))
1836 vel = numpy.zeros((nPairs,3,nHeights))
1837 dist1 = numpy.reshape(dist, (dist.size,1))
1837 dist1 = numpy.reshape(dist, (dist.size,1))
1838
1838
1839 angCos = numpy.cos(ang)
1839 angCos = numpy.cos(ang)
1840 angSin = numpy.sin(ang)
1840 angSin = numpy.sin(ang)
1841
1841
1842 vel0 = dist1*tau1/(2*tau2**2)
1842 vel0 = dist1*tau1/(2*tau2**2)
1843 vel[:,0,:] = (vel0*angCos).sum(axis = 1)
1843 vel[:,0,:] = (vel0*angCos).sum(axis = 1)
1844 vel[:,1,:] = (vel0*angSin).sum(axis = 1)
1844 vel[:,1,:] = (vel0*angSin).sum(axis = 1)
1845
1845
1846 ind = numpy.where(numpy.isinf(vel))
1846 ind = numpy.where(numpy.isinf(vel))
1847 vel[ind] = numpy.nan
1847 vel[ind] = numpy.nan
1848
1848
1849 return vel
1849 return vel
1850
1850
1851 # def __getPairsAutoCorr(self, pairsList, nChannels):
1851 # def __getPairsAutoCorr(self, pairsList, nChannels):
1852 #
1852 #
1853 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
1853 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
1854 #
1854 #
1855 # for l in range(len(pairsList)):
1855 # for l in range(len(pairsList)):
1856 # firstChannel = pairsList[l][0]
1856 # firstChannel = pairsList[l][0]
1857 # secondChannel = pairsList[l][1]
1857 # secondChannel = pairsList[l][1]
1858 #
1858 #
1859 # #Obteniendo pares de Autocorrelacion
1859 # #Obteniendo pares de Autocorrelacion
1860 # if firstChannel == secondChannel:
1860 # if firstChannel == secondChannel:
1861 # pairsAutoCorr[firstChannel] = int(l)
1861 # pairsAutoCorr[firstChannel] = int(l)
1862 #
1862 #
1863 # pairsAutoCorr = pairsAutoCorr.astype(int)
1863 # pairsAutoCorr = pairsAutoCorr.astype(int)
1864 #
1864 #
1865 # pairsCrossCorr = range(len(pairsList))
1865 # pairsCrossCorr = range(len(pairsList))
1866 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
1866 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
1867 #
1867 #
1868 # return pairsAutoCorr, pairsCrossCorr
1868 # return pairsAutoCorr, pairsCrossCorr
1869
1869
1870 # def techniqueSA(self, pairsSelected, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, lagTRange, correctFactor):
1870 # def techniqueSA(self, pairsSelected, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, lagTRange, correctFactor):
1871 def techniqueSA(self, kwargs):
1871 def techniqueSA(self, kwargs):
1872
1872
1873 """
1873 """
1874 Function that implements Spaced Antenna (SA) technique.
1874 Function that implements Spaced Antenna (SA) technique.
1875
1875
1876 Input: Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
1876 Input: Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
1877 Direction correction (if necessary), Ranges and SNR
1877 Direction correction (if necessary), Ranges and SNR
1878
1878
1879 Output: Winds estimation (Zonal, Meridional and Vertical)
1879 Output: Winds estimation (Zonal, Meridional and Vertical)
1880
1880
1881 Parameters affected: Winds
1881 Parameters affected: Winds
1882 """
1882 """
1883 position_x = kwargs['positionX']
1883 position_x = kwargs['positionX']
1884 position_y = kwargs['positionY']
1884 position_y = kwargs['positionY']
1885 azimuth = kwargs['azimuth']
1885 azimuth = kwargs['azimuth']
1886
1886
1887 if 'correctFactor' in kwargs:
1887 if 'correctFactor' in kwargs:
1888 correctFactor = kwargs['correctFactor']
1888 correctFactor = kwargs['correctFactor']
1889 else:
1889 else:
1890 correctFactor = 1
1890 correctFactor = 1
1891
1891
1892 groupList = kwargs['groupList']
1892 groupList = kwargs['groupList']
1893 pairs_ccf = groupList[1]
1893 pairs_ccf = groupList[1]
1894 tau = kwargs['tau']
1894 tau = kwargs['tau']
1895 _lambda = kwargs['_lambda']
1895 _lambda = kwargs['_lambda']
1896
1896
1897 #Cross Correlation pairs obtained
1897 #Cross Correlation pairs obtained
1898 # pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairssList, nChannels)
1898 # pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairssList, nChannels)
1899 # pairsArray = numpy.array(pairsList)[pairsCrossCorr]
1899 # pairsArray = numpy.array(pairsList)[pairsCrossCorr]
1900 # pairsSelArray = numpy.array(pairsSelected)
1900 # pairsSelArray = numpy.array(pairsSelected)
1901 # pairs = []
1901 # pairs = []
1902 #
1902 #
1903 # #Wind estimation pairs obtained
1903 # #Wind estimation pairs obtained
1904 # for i in range(pairsSelArray.shape[0]/2):
1904 # for i in range(pairsSelArray.shape[0]/2):
1905 # ind1 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i], axis = 1))[0][0]
1905 # ind1 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i], axis = 1))[0][0]
1906 # ind2 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i + 1], axis = 1))[0][0]
1906 # ind2 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i + 1], axis = 1))[0][0]
1907 # pairs.append((ind1,ind2))
1907 # pairs.append((ind1,ind2))
1908
1908
1909 indtau = tau.shape[0]/2
1909 indtau = tau.shape[0]/2
1910 tau1 = tau[:indtau,:]
1910 tau1 = tau[:indtau,:]
1911 tau2 = tau[indtau:-1,:]
1911 tau2 = tau[indtau:-1,:]
1912 # tau1 = tau1[pairs,:]
1912 # tau1 = tau1[pairs,:]
1913 # tau2 = tau2[pairs,:]
1913 # tau2 = tau2[pairs,:]
1914 phase1 = tau[-1,:]
1914 phase1 = tau[-1,:]
1915
1915
1916 #---------------------------------------------------------------------
1916 #---------------------------------------------------------------------
1917 #Metodo Directo
1917 #Metodo Directo
1918 distx, disty, dist, ang = self.__calculateDistance(position_x, position_y, pairs_ccf,azimuth)
1918 distx, disty, dist, ang = self.__calculateDistance(position_x, position_y, pairs_ccf,azimuth)
1919 winds = self.__calculateVelHorDir(dist, tau1, tau2, ang)
1919 winds = self.__calculateVelHorDir(dist, tau1, tau2, ang)
1920 winds = stats.nanmean(winds, axis=0)
1920 winds = stats.nanmean(winds, axis=0)
1921 #---------------------------------------------------------------------
1921 #---------------------------------------------------------------------
1922 #Metodo General
1922 #Metodo General
1923 # distx, disty, dist = self.calculateDistance(position_x,position_y,pairsCrossCorr, pairsList, azimuth)
1923 # distx, disty, dist = self.calculateDistance(position_x,position_y,pairsCrossCorr, pairsList, azimuth)
1924 # #Calculo Coeficientes de Funcion de Correlacion
1924 # #Calculo Coeficientes de Funcion de Correlacion
1925 # F,G,A,B,H = self.calculateCoef(tau1,tau2,distx,disty,n)
1925 # F,G,A,B,H = self.calculateCoef(tau1,tau2,distx,disty,n)
1926 # #Calculo de Velocidades
1926 # #Calculo de Velocidades
1927 # winds = self.calculateVelUV(F,G,A,B,H)
1927 # winds = self.calculateVelUV(F,G,A,B,H)
1928
1928
1929 #---------------------------------------------------------------------
1929 #---------------------------------------------------------------------
1930 winds[2,:] = self.__calculateVelVer(phase1, lagTRange, _lambda)
1930 winds[2,:] = self.__calculateVelVer(phase1, lagTRange, _lambda)
1931 winds = correctFactor*winds
1931 winds = correctFactor*winds
1932 return winds
1932 return winds
1933
1933
1934 def __checkTime(self, currentTime, paramInterval, outputInterval):
1934 def __checkTime(self, currentTime, paramInterval, outputInterval):
1935
1935
1936 dataTime = currentTime + paramInterval
1936 dataTime = currentTime + paramInterval
1937 deltaTime = dataTime - self.__initime
1937 deltaTime = dataTime - self.__initime
1938
1938
1939 if deltaTime >= outputInterval or deltaTime < 0:
1939 if deltaTime >= outputInterval or deltaTime < 0:
1940 self.__dataReady = True
1940 self.__dataReady = True
1941 return
1941 return
1942
1942
1943 def techniqueMeteors(self, arrayMeteor, meteorThresh, heightMin, heightMax):
1943 def techniqueMeteors(self, arrayMeteor, meteorThresh, heightMin, heightMax):
1944 '''
1944 '''
1945 Function that implements winds estimation technique with detected meteors.
1945 Function that implements winds estimation technique with detected meteors.
1946
1946
1947 Input: Detected meteors, Minimum meteor quantity to wind estimation
1947 Input: Detected meteors, Minimum meteor quantity to wind estimation
1948
1948
1949 Output: Winds estimation (Zonal and Meridional)
1949 Output: Winds estimation (Zonal and Meridional)
1950
1950
1951 Parameters affected: Winds
1951 Parameters affected: Winds
1952 '''
1952 '''
1953 #Settings
1953 #Settings
1954 nInt = (heightMax - heightMin)/2
1954 nInt = (heightMax - heightMin)/2
1955 nInt = int(nInt)
1955 nInt = int(nInt)
1956 winds = numpy.zeros((2,nInt))*numpy.nan
1956 winds = numpy.zeros((2,nInt))*numpy.nan
1957
1957
1958 #Filter errors
1958 #Filter errors
1959 error = numpy.where(arrayMeteor[:,-1] == 0)[0]
1959 error = numpy.where(arrayMeteor[:,-1] == 0)[0]
1960 finalMeteor = arrayMeteor[error,:]
1960 finalMeteor = arrayMeteor[error,:]
1961
1961
1962 #Meteor Histogram
1962 #Meteor Histogram
1963 finalHeights = finalMeteor[:,2]
1963 finalHeights = finalMeteor[:,2]
1964 hist = numpy.histogram(finalHeights, bins = nInt, range = (heightMin,heightMax))
1964 hist = numpy.histogram(finalHeights, bins = nInt, range = (heightMin,heightMax))
1965 nMeteorsPerI = hist[0]
1965 nMeteorsPerI = hist[0]
1966 heightPerI = hist[1]
1966 heightPerI = hist[1]
1967
1967
1968 #Sort of meteors
1968 #Sort of meteors
1969 indSort = finalHeights.argsort()
1969 indSort = finalHeights.argsort()
1970 finalMeteor2 = finalMeteor[indSort,:]
1970 finalMeteor2 = finalMeteor[indSort,:]
1971
1971
1972 # Calculating winds
1972 # Calculating winds
1973 ind1 = 0
1973 ind1 = 0
1974 ind2 = 0
1974 ind2 = 0
1975
1975
1976 for i in range(nInt):
1976 for i in range(nInt):
1977 nMet = nMeteorsPerI[i]
1977 nMet = nMeteorsPerI[i]
1978 ind1 = ind2
1978 ind1 = ind2
1979 ind2 = ind1 + nMet
1979 ind2 = ind1 + nMet
1980
1980
1981 meteorAux = finalMeteor2[ind1:ind2,:]
1981 meteorAux = finalMeteor2[ind1:ind2,:]
1982
1982
1983 if meteorAux.shape[0] >= meteorThresh:
1983 if meteorAux.shape[0] >= meteorThresh:
1984 vel = meteorAux[:, 6]
1984 vel = meteorAux[:, 6]
1985 zen = meteorAux[:, 4]*numpy.pi/180
1985 zen = meteorAux[:, 4]*numpy.pi/180
1986 azim = meteorAux[:, 3]*numpy.pi/180
1986 azim = meteorAux[:, 3]*numpy.pi/180
1987
1987
1988 n = numpy.cos(zen)
1988 n = numpy.cos(zen)
1989 # m = (1 - n**2)/(1 - numpy.tan(azim)**2)
1989 # m = (1 - n**2)/(1 - numpy.tan(azim)**2)
1990 # l = m*numpy.tan(azim)
1990 # l = m*numpy.tan(azim)
1991 l = numpy.sin(zen)*numpy.sin(azim)
1991 l = numpy.sin(zen)*numpy.sin(azim)
1992 m = numpy.sin(zen)*numpy.cos(azim)
1992 m = numpy.sin(zen)*numpy.cos(azim)
1993
1993
1994 A = numpy.vstack((l, m)).transpose()
1994 A = numpy.vstack((l, m)).transpose()
1995 A1 = numpy.dot(numpy.linalg.inv( numpy.dot(A.transpose(),A) ),A.transpose())
1995 A1 = numpy.dot(numpy.linalg.inv( numpy.dot(A.transpose(),A) ),A.transpose())
1996 windsAux = numpy.dot(A1, vel)
1996 windsAux = numpy.dot(A1, vel)
1997
1997
1998 winds[0,i] = windsAux[0]
1998 winds[0,i] = windsAux[0]
1999 winds[1,i] = windsAux[1]
1999 winds[1,i] = windsAux[1]
2000
2000
2001 return winds, heightPerI[:-1]
2001 return winds, heightPerI[:-1]
2002
2002
2003 def techniqueNSM_SA(self, **kwargs):
2003 def techniqueNSM_SA(self, **kwargs):
2004 metArray = kwargs['metArray']
2004 metArray = kwargs['metArray']
2005 heightList = kwargs['heightList']
2005 heightList = kwargs['heightList']
2006 timeList = kwargs['timeList']
2006 timeList = kwargs['timeList']
2007
2007
2008 rx_location = kwargs['rx_location']
2008 rx_location = kwargs['rx_location']
2009 groupList = kwargs['groupList']
2009 groupList = kwargs['groupList']
2010 azimuth = kwargs['azimuth']
2010 azimuth = kwargs['azimuth']
2011 dfactor = kwargs['dfactor']
2011 dfactor = kwargs['dfactor']
2012 k = kwargs['k']
2012 k = kwargs['k']
2013
2013
2014 azimuth1, dist = self.__calculateAzimuth1(rx_location, groupList, azimuth)
2014 azimuth1, dist = self.__calculateAzimuth1(rx_location, groupList, azimuth)
2015 d = dist*dfactor
2015 d = dist*dfactor
2016 #Phase calculation
2016 #Phase calculation
2017 metArray1 = self.__getPhaseSlope(metArray, heightList, timeList)
2017 metArray1 = self.__getPhaseSlope(metArray, heightList, timeList)
2018
2018
2019 metArray1[:,-2] = metArray1[:,-2]*metArray1[:,2]*1000/(k*d[metArray1[:,1].astype(int)]) #angles into velocities
2019 metArray1[:,-2] = metArray1[:,-2]*metArray1[:,2]*1000/(k*d[metArray1[:,1].astype(int)]) #angles into velocities
2020
2020
2021 velEst = numpy.zeros((heightList.size,2))*numpy.nan
2021 velEst = numpy.zeros((heightList.size,2))*numpy.nan
2022 azimuth1 = azimuth1*numpy.pi/180
2022 azimuth1 = azimuth1*numpy.pi/180
2023
2023
2024 for i in range(heightList.size):
2024 for i in range(heightList.size):
2025 h = heightList[i]
2025 h = heightList[i]
2026 indH = numpy.where((metArray1[:,2] == h)&(numpy.abs(metArray1[:,-2]) < 100))[0]
2026 indH = numpy.where((metArray1[:,2] == h)&(numpy.abs(metArray1[:,-2]) < 100))[0]
2027 metHeight = metArray1[indH,:]
2027 metHeight = metArray1[indH,:]
2028 if metHeight.shape[0] >= 2:
2028 if metHeight.shape[0] >= 2:
2029 velAux = numpy.asmatrix(metHeight[:,-2]).T #Radial Velocities
2029 velAux = numpy.asmatrix(metHeight[:,-2]).T #Radial Velocities
2030 iazim = metHeight[:,1].astype(int)
2030 iazim = metHeight[:,1].astype(int)
2031 azimAux = numpy.asmatrix(azimuth1[iazim]).T #Azimuths
2031 azimAux = numpy.asmatrix(azimuth1[iazim]).T #Azimuths
2032 A = numpy.hstack((numpy.cos(azimAux),numpy.sin(azimAux)))
2032 A = numpy.hstack((numpy.cos(azimAux),numpy.sin(azimAux)))
2033 A = numpy.asmatrix(A)
2033 A = numpy.asmatrix(A)
2034 A1 = numpy.linalg.pinv(A.transpose()*A)*A.transpose()
2034 A1 = numpy.linalg.pinv(A.transpose()*A)*A.transpose()
2035 velHor = numpy.dot(A1,velAux)
2035 velHor = numpy.dot(A1,velAux)
2036
2036
2037 velEst[i,:] = numpy.squeeze(velHor)
2037 velEst[i,:] = numpy.squeeze(velHor)
2038 return velEst
2038 return velEst
2039
2039
2040 def __getPhaseSlope(self, metArray, heightList, timeList):
2040 def __getPhaseSlope(self, metArray, heightList, timeList):
2041 meteorList = []
2041 meteorList = []
2042 #utctime sec1 height SNR velRad ph0 ph1 ph2 coh0 coh1 coh2
2042 #utctime sec1 height SNR velRad ph0 ph1 ph2 coh0 coh1 coh2
2043 #Putting back together the meteor matrix
2043 #Putting back together the meteor matrix
2044 utctime = metArray[:,0]
2044 utctime = metArray[:,0]
2045 uniqueTime = numpy.unique(utctime)
2045 uniqueTime = numpy.unique(utctime)
2046
2046
2047 phaseDerThresh = 0.5
2047 phaseDerThresh = 0.5
2048 ippSeconds = timeList[1] - timeList[0]
2048 ippSeconds = timeList[1] - timeList[0]
2049 sec = numpy.where(timeList>1)[0][0]
2049 sec = numpy.where(timeList>1)[0][0]
2050 nPairs = metArray.shape[1] - 6
2050 nPairs = metArray.shape[1] - 6
2051 nHeights = len(heightList)
2051 nHeights = len(heightList)
2052
2052
2053 for t in uniqueTime:
2053 for t in uniqueTime:
2054 metArray1 = metArray[utctime==t,:]
2054 metArray1 = metArray[utctime==t,:]
2055 # phaseDerThresh = numpy.pi/4 #reducir Phase thresh
2055 # phaseDerThresh = numpy.pi/4 #reducir Phase thresh
2056 tmet = metArray1[:,1].astype(int)
2056 tmet = metArray1[:,1].astype(int)
2057 hmet = metArray1[:,2].astype(int)
2057 hmet = metArray1[:,2].astype(int)
2058
2058
2059 metPhase = numpy.zeros((nPairs, heightList.size, timeList.size - 1))
2059 metPhase = numpy.zeros((nPairs, heightList.size, timeList.size - 1))
2060 metPhase[:,:] = numpy.nan
2060 metPhase[:,:] = numpy.nan
2061 metPhase[:,hmet,tmet] = metArray1[:,6:].T
2061 metPhase[:,hmet,tmet] = metArray1[:,6:].T
2062
2062
2063 #Delete short trails
2063 #Delete short trails
2064 metBool = ~numpy.isnan(metPhase[0,:,:])
2064 metBool = ~numpy.isnan(metPhase[0,:,:])
2065 heightVect = numpy.sum(metBool, axis = 1)
2065 heightVect = numpy.sum(metBool, axis = 1)
2066 metBool[heightVect<sec,:] = False
2066 metBool[heightVect<sec,:] = False
2067 metPhase[:,heightVect<sec,:] = numpy.nan
2067 metPhase[:,heightVect<sec,:] = numpy.nan
2068
2068
2069 #Derivative
2069 #Derivative
2070 metDer = numpy.abs(metPhase[:,:,1:] - metPhase[:,:,:-1])
2070 metDer = numpy.abs(metPhase[:,:,1:] - metPhase[:,:,:-1])
2071 phDerAux = numpy.dstack((numpy.full((nPairs,nHeights,1), False, dtype=bool),metDer > phaseDerThresh))
2071 phDerAux = numpy.dstack((numpy.full((nPairs,nHeights,1), False, dtype=bool),metDer > phaseDerThresh))
2072 metPhase[phDerAux] = numpy.nan
2072 metPhase[phDerAux] = numpy.nan
2073
2073
2074 #--------------------------METEOR DETECTION -----------------------------------------
2074 #--------------------------METEOR DETECTION -----------------------------------------
2075 indMet = numpy.where(numpy.any(metBool,axis=1))[0]
2075 indMet = numpy.where(numpy.any(metBool,axis=1))[0]
2076
2076
2077 for p in numpy.arange(nPairs):
2077 for p in numpy.arange(nPairs):
2078 phase = metPhase[p,:,:]
2078 phase = metPhase[p,:,:]
2079 phDer = metDer[p,:,:]
2079 phDer = metDer[p,:,:]
2080
2080
2081 for h in indMet:
2081 for h in indMet:
2082 height = heightList[h]
2082 height = heightList[h]
2083 phase1 = phase[h,:] #82
2083 phase1 = phase[h,:] #82
2084 phDer1 = phDer[h,:]
2084 phDer1 = phDer[h,:]
2085
2085
2086 phase1[~numpy.isnan(phase1)] = numpy.unwrap(phase1[~numpy.isnan(phase1)]) #Unwrap
2086 phase1[~numpy.isnan(phase1)] = numpy.unwrap(phase1[~numpy.isnan(phase1)]) #Unwrap
2087
2087
2088 indValid = numpy.where(~numpy.isnan(phase1))[0]
2088 indValid = numpy.where(~numpy.isnan(phase1))[0]
2089 initMet = indValid[0]
2089 initMet = indValid[0]
2090 endMet = 0
2090 endMet = 0
2091
2091
2092 for i in range(len(indValid)-1):
2092 for i in range(len(indValid)-1):
2093
2093
2094 #Time difference
2094 #Time difference
2095 inow = indValid[i]
2095 inow = indValid[i]
2096 inext = indValid[i+1]
2096 inext = indValid[i+1]
2097 idiff = inext - inow
2097 idiff = inext - inow
2098 #Phase difference
2098 #Phase difference
2099 phDiff = numpy.abs(phase1[inext] - phase1[inow])
2099 phDiff = numpy.abs(phase1[inext] - phase1[inow])
2100
2100
2101 if idiff>sec or phDiff>numpy.pi/4 or inext==indValid[-1]: #End of Meteor
2101 if idiff>sec or phDiff>numpy.pi/4 or inext==indValid[-1]: #End of Meteor
2102 sizeTrail = inow - initMet + 1
2102 sizeTrail = inow - initMet + 1
2103 if sizeTrail>3*sec: #Too short meteors
2103 if sizeTrail>3*sec: #Too short meteors
2104 x = numpy.arange(initMet,inow+1)*ippSeconds
2104 x = numpy.arange(initMet,inow+1)*ippSeconds
2105 y = phase1[initMet:inow+1]
2105 y = phase1[initMet:inow+1]
2106 ynnan = ~numpy.isnan(y)
2106 ynnan = ~numpy.isnan(y)
2107 x = x[ynnan]
2107 x = x[ynnan]
2108 y = y[ynnan]
2108 y = y[ynnan]
2109 slope, intercept, r_value, p_value, std_err = stats.linregress(x,y)
2109 slope, intercept, r_value, p_value, std_err = stats.linregress(x,y)
2110 ylin = x*slope + intercept
2110 ylin = x*slope + intercept
2111 rsq = r_value**2
2111 rsq = r_value**2
2112 if rsq > 0.5:
2112 if rsq > 0.5:
2113 vel = slope#*height*1000/(k*d)
2113 vel = slope#*height*1000/(k*d)
2114 estAux = numpy.array([utctime,p,height, vel, rsq])
2114 estAux = numpy.array([utctime,p,height, vel, rsq])
2115 meteorList.append(estAux)
2115 meteorList.append(estAux)
2116 initMet = inext
2116 initMet = inext
2117 metArray2 = numpy.array(meteorList)
2117 metArray2 = numpy.array(meteorList)
2118
2118
2119 return metArray2
2119 return metArray2
2120
2120
2121 def __calculateAzimuth1(self, rx_location, pairslist, azimuth0):
2121 def __calculateAzimuth1(self, rx_location, pairslist, azimuth0):
2122
2122
2123 azimuth1 = numpy.zeros(len(pairslist))
2123 azimuth1 = numpy.zeros(len(pairslist))
2124 dist = numpy.zeros(len(pairslist))
2124 dist = numpy.zeros(len(pairslist))
2125
2125
2126 for i in range(len(rx_location)):
2126 for i in range(len(rx_location)):
2127 ch0 = pairslist[i][0]
2127 ch0 = pairslist[i][0]
2128 ch1 = pairslist[i][1]
2128 ch1 = pairslist[i][1]
2129
2129
2130 diffX = rx_location[ch0][0] - rx_location[ch1][0]
2130 diffX = rx_location[ch0][0] - rx_location[ch1][0]
2131 diffY = rx_location[ch0][1] - rx_location[ch1][1]
2131 diffY = rx_location[ch0][1] - rx_location[ch1][1]
2132 azimuth1[i] = numpy.arctan2(diffY,diffX)*180/numpy.pi
2132 azimuth1[i] = numpy.arctan2(diffY,diffX)*180/numpy.pi
2133 dist[i] = numpy.sqrt(diffX**2 + diffY**2)
2133 dist[i] = numpy.sqrt(diffX**2 + diffY**2)
2134
2134
2135 azimuth1 -= azimuth0
2135 azimuth1 -= azimuth0
2136 return azimuth1, dist
2136 return azimuth1, dist
2137
2137
2138 def techniqueNSM_DBS(self, **kwargs):
2138 def techniqueNSM_DBS(self, **kwargs):
2139 metArray = kwargs['metArray']
2139 metArray = kwargs['metArray']
2140 heightList = kwargs['heightList']
2140 heightList = kwargs['heightList']
2141 timeList = kwargs['timeList']
2141 timeList = kwargs['timeList']
2142 azimuth = kwargs['azimuth']
2142 azimuth = kwargs['azimuth']
2143 theta_x = numpy.array(kwargs['theta_x'])
2143 theta_x = numpy.array(kwargs['theta_x'])
2144 theta_y = numpy.array(kwargs['theta_y'])
2144 theta_y = numpy.array(kwargs['theta_y'])
2145
2145
2146 utctime = metArray[:,0]
2146 utctime = metArray[:,0]
2147 cmet = metArray[:,1].astype(int)
2147 cmet = metArray[:,1].astype(int)
2148 hmet = metArray[:,3].astype(int)
2148 hmet = metArray[:,3].astype(int)
2149 SNRmet = metArray[:,4]
2149 SNRmet = metArray[:,4]
2150 vmet = metArray[:,5]
2150 vmet = metArray[:,5]
2151 spcmet = metArray[:,6]
2151 spcmet = metArray[:,6]
2152
2152
2153 nChan = numpy.max(cmet) + 1
2153 nChan = numpy.max(cmet) + 1
2154 nHeights = len(heightList)
2154 nHeights = len(heightList)
2155
2155
2156 azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw = self.__calculateAngles(theta_x, theta_y, azimuth)
2156 azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw = self.__calculateAngles(theta_x, theta_y, azimuth)
2157 hmet = heightList[hmet]
2157 hmet = heightList[hmet]
2158 h1met = hmet*numpy.cos(zenith_arr[cmet]) #Corrected heights
2158 h1met = hmet*numpy.cos(zenith_arr[cmet]) #Corrected heights
2159
2159
2160 velEst = numpy.zeros((heightList.size,2))*numpy.nan
2160 velEst = numpy.zeros((heightList.size,2))*numpy.nan
2161
2161
2162 for i in range(nHeights - 1):
2162 for i in range(nHeights - 1):
2163 hmin = heightList[i]
2163 hmin = heightList[i]
2164 hmax = heightList[i + 1]
2164 hmax = heightList[i + 1]
2165
2165
2166 thisH = (h1met>=hmin) & (h1met<hmax) & (cmet!=2) & (SNRmet>8) & (vmet<50) & (spcmet<10)
2166 thisH = (h1met>=hmin) & (h1met<hmax) & (cmet!=2) & (SNRmet>8) & (vmet<50) & (spcmet<10)
2167 indthisH = numpy.where(thisH)
2167 indthisH = numpy.where(thisH)
2168
2168
2169 if numpy.size(indthisH) > 3:
2169 if numpy.size(indthisH) > 3:
2170
2170
2171 vel_aux = vmet[thisH]
2171 vel_aux = vmet[thisH]
2172 chan_aux = cmet[thisH]
2172 chan_aux = cmet[thisH]
2173 cosu_aux = dir_cosu[chan_aux]
2173 cosu_aux = dir_cosu[chan_aux]
2174 cosv_aux = dir_cosv[chan_aux]
2174 cosv_aux = dir_cosv[chan_aux]
2175 cosw_aux = dir_cosw[chan_aux]
2175 cosw_aux = dir_cosw[chan_aux]
2176
2176
2177 nch = numpy.size(numpy.unique(chan_aux))
2177 nch = numpy.size(numpy.unique(chan_aux))
2178 if nch > 1:
2178 if nch > 1:
2179 A = self.__calculateMatA(cosu_aux, cosv_aux, cosw_aux, True)
2179 A = self.__calculateMatA(cosu_aux, cosv_aux, cosw_aux, True)
2180 velEst[i,:] = numpy.dot(A,vel_aux)
2180 velEst[i,:] = numpy.dot(A,vel_aux)
2181
2181
2182 return velEst
2182 return velEst
2183
2183
2184 def run(self, dataOut, technique, nHours=1, hmin=70, hmax=110, **kwargs):
2184 def run(self, dataOut, technique, nHours=1, hmin=70, hmax=110, **kwargs):
2185
2185
2186 param = dataOut.data_param
2186 param = dataOut.data_param
2187 if dataOut.abscissaList != None:
2187 if dataOut.abscissaList != None:
2188 absc = dataOut.abscissaList[:-1]
2188 absc = dataOut.abscissaList[:-1]
2189 # noise = dataOut.noise
2189 # noise = dataOut.noise
2190 heightList = dataOut.heightList
2190 heightList = dataOut.heightList
2191 SNR = dataOut.data_snr
2191 SNR = dataOut.data_snr
2192
2192
2193 if technique == 'DBS':
2193 if technique == 'DBS':
2194
2194
2195 kwargs['velRadial'] = param[:,1,:] #Radial velocity
2195 kwargs['velRadial'] = param[:,1,:] #Radial velocity
2196 kwargs['heightList'] = heightList
2196 kwargs['heightList'] = heightList
2197 kwargs['SNR'] = SNR
2197 kwargs['SNR'] = SNR
2198
2198
2199 dataOut.data_output, dataOut.heightList, dataOut.data_snr = self.techniqueDBS(kwargs) #DBS Function
2199 dataOut.data_output, dataOut.heightList, dataOut.data_snr = self.techniqueDBS(kwargs) #DBS Function
2200 dataOut.utctimeInit = dataOut.utctime
2200 dataOut.utctimeInit = dataOut.utctime
2201 dataOut.outputInterval = dataOut.paramInterval
2201 dataOut.outputInterval = dataOut.paramInterval
2202
2202
2203 elif technique == 'SA':
2203 elif technique == 'SA':
2204
2204
2205 #Parameters
2205 #Parameters
2206 # position_x = kwargs['positionX']
2206 # position_x = kwargs['positionX']
2207 # position_y = kwargs['positionY']
2207 # position_y = kwargs['positionY']
2208 # azimuth = kwargs['azimuth']
2208 # azimuth = kwargs['azimuth']
2209 #
2209 #
2210 # if kwargs.has_key('crosspairsList'):
2210 # if kwargs.has_key('crosspairsList'):
2211 # pairs = kwargs['crosspairsList']
2211 # pairs = kwargs['crosspairsList']
2212 # else:
2212 # else:
2213 # pairs = None
2213 # pairs = None
2214 #
2214 #
2215 # if kwargs.has_key('correctFactor'):
2215 # if kwargs.has_key('correctFactor'):
2216 # correctFactor = kwargs['correctFactor']
2216 # correctFactor = kwargs['correctFactor']
2217 # else:
2217 # else:
2218 # correctFactor = 1
2218 # correctFactor = 1
2219
2219
2220 # tau = dataOut.data_param
2220 # tau = dataOut.data_param
2221 # _lambda = dataOut.C/dataOut.frequency
2221 # _lambda = dataOut.C/dataOut.frequency
2222 # pairsList = dataOut.groupList
2222 # pairsList = dataOut.groupList
2223 # nChannels = dataOut.nChannels
2223 # nChannels = dataOut.nChannels
2224
2224
2225 kwargs['groupList'] = dataOut.groupList
2225 kwargs['groupList'] = dataOut.groupList
2226 kwargs['tau'] = dataOut.data_param
2226 kwargs['tau'] = dataOut.data_param
2227 kwargs['_lambda'] = dataOut.C/dataOut.frequency
2227 kwargs['_lambda'] = dataOut.C/dataOut.frequency
2228 # dataOut.data_output = self.techniqueSA(pairs, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, absc, correctFactor)
2228 # dataOut.data_output = self.techniqueSA(pairs, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, absc, correctFactor)
2229 dataOut.data_output = self.techniqueSA(kwargs)
2229 dataOut.data_output = self.techniqueSA(kwargs)
2230 dataOut.utctimeInit = dataOut.utctime
2230 dataOut.utctimeInit = dataOut.utctime
2231 dataOut.outputInterval = dataOut.timeInterval
2231 dataOut.outputInterval = dataOut.timeInterval
2232
2232
2233 elif technique == 'Meteors':
2233 elif technique == 'Meteors':
2234 dataOut.flagNoData = True
2234 dataOut.flagNoData = True
2235 self.__dataReady = False
2235 self.__dataReady = False
2236
2236
2237 if 'nHours' in kwargs:
2237 if 'nHours' in kwargs:
2238 nHours = kwargs['nHours']
2238 nHours = kwargs['nHours']
2239 else:
2239 else:
2240 nHours = 1
2240 nHours = 1
2241
2241
2242 if 'meteorsPerBin' in kwargs:
2242 if 'meteorsPerBin' in kwargs:
2243 meteorThresh = kwargs['meteorsPerBin']
2243 meteorThresh = kwargs['meteorsPerBin']
2244 else:
2244 else:
2245 meteorThresh = 6
2245 meteorThresh = 6
2246
2246
2247 if 'hmin' in kwargs:
2247 if 'hmin' in kwargs:
2248 hmin = kwargs['hmin']
2248 hmin = kwargs['hmin']
2249 else: hmin = 70
2249 else: hmin = 70
2250 if 'hmax' in kwargs:
2250 if 'hmax' in kwargs:
2251 hmax = kwargs['hmax']
2251 hmax = kwargs['hmax']
2252 else: hmax = 110
2252 else: hmax = 110
2253
2253
2254 dataOut.outputInterval = nHours*3600
2254 dataOut.outputInterval = nHours*3600
2255
2255
2256 if self.__isConfig == False:
2256 if self.__isConfig == False:
2257 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
2257 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
2258 #Get Initial LTC time
2258 #Get Initial LTC time
2259 self.__initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
2259 self.__initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
2260 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
2260 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
2261
2261
2262 self.__isConfig = True
2262 self.__isConfig = True
2263
2263
2264 if self.__buffer is None:
2264 if self.__buffer is None:
2265 self.__buffer = dataOut.data_param
2265 self.__buffer = dataOut.data_param
2266 self.__firstdata = copy.copy(dataOut)
2266 self.__firstdata = copy.copy(dataOut)
2267
2267
2268 else:
2268 else:
2269 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
2269 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
2270
2270
2271 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
2271 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
2272
2272
2273 if self.__dataReady:
2273 if self.__dataReady:
2274 dataOut.utctimeInit = self.__initime
2274 dataOut.utctimeInit = self.__initime
2275
2275
2276 self.__initime += dataOut.outputInterval #to erase time offset
2276 self.__initime += dataOut.outputInterval #to erase time offset
2277
2277
2278 dataOut.data_output, dataOut.heightList = self.techniqueMeteors(self.__buffer, meteorThresh, hmin, hmax)
2278 dataOut.data_output, dataOut.heightList = self.techniqueMeteors(self.__buffer, meteorThresh, hmin, hmax)
2279 dataOut.flagNoData = False
2279 dataOut.flagNoData = False
2280 self.__buffer = None
2280 self.__buffer = None
2281
2281
2282 elif technique == 'Meteors1':
2282 elif technique == 'Meteors1':
2283 dataOut.flagNoData = True
2283 dataOut.flagNoData = True
2284 self.__dataReady = False
2284 self.__dataReady = False
2285
2285
2286 if 'nMins' in kwargs:
2286 if 'nMins' in kwargs:
2287 nMins = kwargs['nMins']
2287 nMins = kwargs['nMins']
2288 else: nMins = 20
2288 else: nMins = 20
2289 if 'rx_location' in kwargs:
2289 if 'rx_location' in kwargs:
2290 rx_location = kwargs['rx_location']
2290 rx_location = kwargs['rx_location']
2291 else: rx_location = [(0,1),(1,1),(1,0)]
2291 else: rx_location = [(0,1),(1,1),(1,0)]
2292 if 'azimuth' in kwargs:
2292 if 'azimuth' in kwargs:
2293 azimuth = kwargs['azimuth']
2293 azimuth = kwargs['azimuth']
2294 else: azimuth = 51.06
2294 else: azimuth = 51.06
2295 if 'dfactor' in kwargs:
2295 if 'dfactor' in kwargs:
2296 dfactor = kwargs['dfactor']
2296 dfactor = kwargs['dfactor']
2297 if 'mode' in kwargs:
2297 if 'mode' in kwargs:
2298 mode = kwargs['mode']
2298 mode = kwargs['mode']
2299 if 'theta_x' in kwargs:
2299 if 'theta_x' in kwargs:
2300 theta_x = kwargs['theta_x']
2300 theta_x = kwargs['theta_x']
2301 if 'theta_y' in kwargs:
2301 if 'theta_y' in kwargs:
2302 theta_y = kwargs['theta_y']
2302 theta_y = kwargs['theta_y']
2303 else: mode = 'SA'
2303 else: mode = 'SA'
2304
2304
2305 #Borrar luego esto
2305 #Borrar luego esto
2306 if dataOut.groupList is None:
2306 if dataOut.groupList is None:
2307 dataOut.groupList = [(0,1),(0,2),(1,2)]
2307 dataOut.groupList = [(0,1),(0,2),(1,2)]
2308 groupList = dataOut.groupList
2308 groupList = dataOut.groupList
2309 C = 3e8
2309 C = 3e8
2310 freq = 50e6
2310 freq = 50e6
2311 lamb = C/freq
2311 lamb = C/freq
2312 k = 2*numpy.pi/lamb
2312 k = 2*numpy.pi/lamb
2313
2313
2314 timeList = dataOut.abscissaList
2314 timeList = dataOut.abscissaList
2315 heightList = dataOut.heightList
2315 heightList = dataOut.heightList
2316
2316
2317 if self.__isConfig == False:
2317 if self.__isConfig == False:
2318 dataOut.outputInterval = nMins*60
2318 dataOut.outputInterval = nMins*60
2319 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
2319 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
2320 #Get Initial LTC time
2320 #Get Initial LTC time
2321 initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
2321 initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
2322 minuteAux = initime.minute
2322 minuteAux = initime.minute
2323 minuteNew = int(numpy.floor(minuteAux/nMins)*nMins)
2323 minuteNew = int(numpy.floor(minuteAux/nMins)*nMins)
2324 self.__initime = (initime.replace(minute = minuteNew, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
2324 self.__initime = (initime.replace(minute = minuteNew, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
2325
2325
2326 self.__isConfig = True
2326 self.__isConfig = True
2327
2327
2328 if self.__buffer is None:
2328 if self.__buffer is None:
2329 self.__buffer = dataOut.data_param
2329 self.__buffer = dataOut.data_param
2330 self.__firstdata = copy.copy(dataOut)
2330 self.__firstdata = copy.copy(dataOut)
2331
2331
2332 else:
2332 else:
2333 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
2333 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
2334
2334
2335 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
2335 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
2336
2336
2337 if self.__dataReady:
2337 if self.__dataReady:
2338 dataOut.utctimeInit = self.__initime
2338 dataOut.utctimeInit = self.__initime
2339 self.__initime += dataOut.outputInterval #to erase time offset
2339 self.__initime += dataOut.outputInterval #to erase time offset
2340
2340
2341 metArray = self.__buffer
2341 metArray = self.__buffer
2342 if mode == 'SA':
2342 if mode == 'SA':
2343 dataOut.data_output = self.techniqueNSM_SA(rx_location=rx_location, groupList=groupList, azimuth=azimuth, dfactor=dfactor, k=k,metArray=metArray, heightList=heightList,timeList=timeList)
2343 dataOut.data_output = self.techniqueNSM_SA(rx_location=rx_location, groupList=groupList, azimuth=azimuth, dfactor=dfactor, k=k,metArray=metArray, heightList=heightList,timeList=timeList)
2344 elif mode == 'DBS':
2344 elif mode == 'DBS':
2345 dataOut.data_output = self.techniqueNSM_DBS(metArray=metArray,heightList=heightList,timeList=timeList, azimuth=azimuth, theta_x=theta_x, theta_y=theta_y)
2345 dataOut.data_output = self.techniqueNSM_DBS(metArray=metArray,heightList=heightList,timeList=timeList, azimuth=azimuth, theta_x=theta_x, theta_y=theta_y)
2346 dataOut.data_output = dataOut.data_output.T
2346 dataOut.data_output = dataOut.data_output.T
2347 dataOut.flagNoData = False
2347 dataOut.flagNoData = False
2348 self.__buffer = None
2348 self.__buffer = None
2349
2349
2350 return
2350 return
2351
2351
2352 class EWDriftsEstimation(Operation):
2352 class EWDriftsEstimation(Operation):
2353
2353
2354 def __init__(self):
2354 def __init__(self):
2355 Operation.__init__(self)
2355 Operation.__init__(self)
2356
2356
2357 def __correctValues(self, heiRang, phi, velRadial, SNR):
2357 def __correctValues(self, heiRang, phi, velRadial, SNR):
2358 listPhi = phi.tolist()
2358 listPhi = phi.tolist()
2359 maxid = listPhi.index(max(listPhi))
2359 maxid = listPhi.index(max(listPhi))
2360 minid = listPhi.index(min(listPhi))
2360 minid = listPhi.index(min(listPhi))
2361
2361
2362 rango = list(range(len(phi)))
2362 rango = list(range(len(phi)))
2363 # rango = numpy.delete(rango,maxid)
2363 # rango = numpy.delete(rango,maxid)
2364
2364
2365 heiRang1 = heiRang*math.cos(phi[maxid])
2365 heiRang1 = heiRang*math.cos(phi[maxid])
2366 heiRangAux = heiRang*math.cos(phi[minid])
2366 heiRangAux = heiRang*math.cos(phi[minid])
2367 indOut = (heiRang1 < heiRangAux[0]).nonzero()
2367 indOut = (heiRang1 < heiRangAux[0]).nonzero()
2368 heiRang1 = numpy.delete(heiRang1,indOut)
2368 heiRang1 = numpy.delete(heiRang1,indOut)
2369
2369
2370 velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
2370 velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
2371 SNR1 = numpy.zeros([len(phi),len(heiRang1)])
2371 SNR1 = numpy.zeros([len(phi),len(heiRang1)])
2372
2372
2373 for i in rango:
2373 for i in rango:
2374 x = heiRang*math.cos(phi[i])
2374 x = heiRang*math.cos(phi[i])
2375 y1 = velRadial[i,:]
2375 y1 = velRadial[i,:]
2376 f1 = interpolate.interp1d(x,y1,kind = 'cubic')
2376 f1 = interpolate.interp1d(x,y1,kind = 'cubic')
2377
2377
2378 x1 = heiRang1
2378 x1 = heiRang1
2379 y11 = f1(x1)
2379 y11 = f1(x1)
2380
2380
2381 y2 = SNR[i,:]
2381 y2 = SNR[i,:]
2382 f2 = interpolate.interp1d(x,y2,kind = 'cubic')
2382 f2 = interpolate.interp1d(x,y2,kind = 'cubic')
2383 y21 = f2(x1)
2383 y21 = f2(x1)
2384
2384
2385 velRadial1[i,:] = y11
2385 velRadial1[i,:] = y11
2386 SNR1[i,:] = y21
2386 SNR1[i,:] = y21
2387
2387
2388 return heiRang1, velRadial1, SNR1
2388 return heiRang1, velRadial1, SNR1
2389
2389
2390 def run(self, dataOut, zenith, zenithCorrection):
2390 def run(self, dataOut, zenith, zenithCorrection):
2391 heiRang = dataOut.heightList
2391 heiRang = dataOut.heightList
2392 velRadial = dataOut.data_param[:,3,:]
2392 velRadial = dataOut.data_param[:,3,:]
2393 SNR = dataOut.data_snr
2393 SNR = dataOut.data_snr
2394
2394
2395 zenith = numpy.array(zenith)
2395 zenith = numpy.array(zenith)
2396 zenith -= zenithCorrection
2396 zenith -= zenithCorrection
2397 zenith *= numpy.pi/180
2397 zenith *= numpy.pi/180
2398
2398
2399 heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, numpy.abs(zenith), velRadial, SNR)
2399 heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, numpy.abs(zenith), velRadial, SNR)
2400
2400
2401 alp = zenith[0]
2401 alp = zenith[0]
2402 bet = zenith[1]
2402 bet = zenith[1]
2403
2403
2404 w_w = velRadial1[0,:]
2404 w_w = velRadial1[0,:]
2405 w_e = velRadial1[1,:]
2405 w_e = velRadial1[1,:]
2406
2406
2407 w = (w_w*numpy.sin(bet) - w_e*numpy.sin(alp))/(numpy.cos(alp)*numpy.sin(bet) - numpy.cos(bet)*numpy.sin(alp))
2407 w = (w_w*numpy.sin(bet) - w_e*numpy.sin(alp))/(numpy.cos(alp)*numpy.sin(bet) - numpy.cos(bet)*numpy.sin(alp))
2408 u = (w_w*numpy.cos(bet) - w_e*numpy.cos(alp))/(numpy.sin(alp)*numpy.cos(bet) - numpy.sin(bet)*numpy.cos(alp))
2408 u = (w_w*numpy.cos(bet) - w_e*numpy.cos(alp))/(numpy.sin(alp)*numpy.cos(bet) - numpy.sin(bet)*numpy.cos(alp))
2409
2409
2410 winds = numpy.vstack((u,w))
2410 winds = numpy.vstack((u,w))
2411
2411
2412 dataOut.heightList = heiRang1
2412 dataOut.heightList = heiRang1
2413 dataOut.data_output = winds
2413 dataOut.data_output = winds
2414 dataOut.data_snr = SNR1
2414 dataOut.data_snr = SNR1
2415
2415
2416 dataOut.utctimeInit = dataOut.utctime
2416 dataOut.utctimeInit = dataOut.utctime
2417 dataOut.outputInterval = dataOut.timeInterval
2417 dataOut.outputInterval = dataOut.timeInterval
2418 return
2418 return
2419
2419
2420 #--------------- Non Specular Meteor ----------------
2420 #--------------- Non Specular Meteor ----------------
2421
2421
2422 class NonSpecularMeteorDetection(Operation):
2422 class NonSpecularMeteorDetection(Operation):
2423
2423
2424 def run(self, dataOut, mode, SNRthresh=8, phaseDerThresh=0.5, cohThresh=0.8, allData = False):
2424 def run(self, dataOut, mode, SNRthresh=8, phaseDerThresh=0.5, cohThresh=0.8, allData = False):
2425 data_acf = dataOut.data_pre[0]
2425 data_acf = dataOut.data_pre[0]
2426 data_ccf = dataOut.data_pre[1]
2426 data_ccf = dataOut.data_pre[1]
2427 pairsList = dataOut.groupList[1]
2427 pairsList = dataOut.groupList[1]
2428
2428
2429 lamb = dataOut.C/dataOut.frequency
2429 lamb = dataOut.C/dataOut.frequency
2430 tSamp = dataOut.ippSeconds*dataOut.nCohInt
2430 tSamp = dataOut.ippSeconds*dataOut.nCohInt
2431 paramInterval = dataOut.paramInterval
2431 paramInterval = dataOut.paramInterval
2432
2432
2433 nChannels = data_acf.shape[0]
2433 nChannels = data_acf.shape[0]
2434 nLags = data_acf.shape[1]
2434 nLags = data_acf.shape[1]
2435 nProfiles = data_acf.shape[2]
2435 nProfiles = data_acf.shape[2]
2436 nHeights = dataOut.nHeights
2436 nHeights = dataOut.nHeights
2437 nCohInt = dataOut.nCohInt
2437 nCohInt = dataOut.nCohInt
2438 sec = numpy.round(nProfiles/dataOut.paramInterval)
2438 sec = numpy.round(nProfiles/dataOut.paramInterval)
2439 heightList = dataOut.heightList
2439 heightList = dataOut.heightList
2440 ippSeconds = dataOut.ippSeconds*dataOut.nCohInt*dataOut.nAvg
2440 ippSeconds = dataOut.ippSeconds*dataOut.nCohInt*dataOut.nAvg
2441 utctime = dataOut.utctime
2441 utctime = dataOut.utctime
2442
2442
2443 dataOut.abscissaList = numpy.arange(0,paramInterval+ippSeconds,ippSeconds)
2443 dataOut.abscissaList = numpy.arange(0,paramInterval+ippSeconds,ippSeconds)
2444
2444
2445 #------------------------ SNR --------------------------------------
2445 #------------------------ SNR --------------------------------------
2446 power = data_acf[:,0,:,:].real
2446 power = data_acf[:,0,:,:].real
2447 noise = numpy.zeros(nChannels)
2447 noise = numpy.zeros(nChannels)
2448 SNR = numpy.zeros(power.shape)
2448 SNR = numpy.zeros(power.shape)
2449 for i in range(nChannels):
2449 for i in range(nChannels):
2450 noise[i] = hildebrand_sekhon(power[i,:], nCohInt)
2450 noise[i] = hildebrand_sekhon(power[i,:], nCohInt)
2451 SNR[i] = (power[i]-noise[i])/noise[i]
2451 SNR[i] = (power[i]-noise[i])/noise[i]
2452 SNRm = numpy.nanmean(SNR, axis = 0)
2452 SNRm = numpy.nanmean(SNR, axis = 0)
2453 SNRdB = 10*numpy.log10(SNR)
2453 SNRdB = 10*numpy.log10(SNR)
2454
2454
2455 if mode == 'SA':
2455 if mode == 'SA':
2456 dataOut.groupList = dataOut.groupList[1]
2456 dataOut.groupList = dataOut.groupList[1]
2457 nPairs = data_ccf.shape[0]
2457 nPairs = data_ccf.shape[0]
2458 #---------------------- Coherence and Phase --------------------------
2458 #---------------------- Coherence and Phase --------------------------
2459 phase = numpy.zeros(data_ccf[:,0,:,:].shape)
2459 phase = numpy.zeros(data_ccf[:,0,:,:].shape)
2460 # phase1 = numpy.copy(phase)
2460 # phase1 = numpy.copy(phase)
2461 coh1 = numpy.zeros(data_ccf[:,0,:,:].shape)
2461 coh1 = numpy.zeros(data_ccf[:,0,:,:].shape)
2462
2462
2463 for p in range(nPairs):
2463 for p in range(nPairs):
2464 ch0 = pairsList[p][0]
2464 ch0 = pairsList[p][0]
2465 ch1 = pairsList[p][1]
2465 ch1 = pairsList[p][1]
2466 ccf = data_ccf[p,0,:,:]/numpy.sqrt(data_acf[ch0,0,:,:]*data_acf[ch1,0,:,:])
2466 ccf = data_ccf[p,0,:,:]/numpy.sqrt(data_acf[ch0,0,:,:]*data_acf[ch1,0,:,:])
2467 phase[p,:,:] = ndimage.median_filter(numpy.angle(ccf), size = (5,1)) #median filter
2467 phase[p,:,:] = ndimage.median_filter(numpy.angle(ccf), size = (5,1)) #median filter
2468 # phase1[p,:,:] = numpy.angle(ccf) #median filter
2468 # phase1[p,:,:] = numpy.angle(ccf) #median filter
2469 coh1[p,:,:] = ndimage.median_filter(numpy.abs(ccf), 5) #median filter
2469 coh1[p,:,:] = ndimage.median_filter(numpy.abs(ccf), 5) #median filter
2470 # coh1[p,:,:] = numpy.abs(ccf) #median filter
2470 # coh1[p,:,:] = numpy.abs(ccf) #median filter
2471 coh = numpy.nanmax(coh1, axis = 0)
2471 coh = numpy.nanmax(coh1, axis = 0)
2472 # struc = numpy.ones((5,1))
2472 # struc = numpy.ones((5,1))
2473 # coh = ndimage.morphology.grey_dilation(coh, size=(10,1))
2473 # coh = ndimage.morphology.grey_dilation(coh, size=(10,1))
2474 #---------------------- Radial Velocity ----------------------------
2474 #---------------------- Radial Velocity ----------------------------
2475 phaseAux = numpy.mean(numpy.angle(data_acf[:,1,:,:]), axis = 0)
2475 phaseAux = numpy.mean(numpy.angle(data_acf[:,1,:,:]), axis = 0)
2476 velRad = phaseAux*lamb/(4*numpy.pi*tSamp)
2476 velRad = phaseAux*lamb/(4*numpy.pi*tSamp)
2477
2477
2478 if allData:
2478 if allData:
2479 boolMetFin = ~numpy.isnan(SNRm)
2479 boolMetFin = ~numpy.isnan(SNRm)
2480 # coh[:-1,:] = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
2480 # coh[:-1,:] = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
2481 else:
2481 else:
2482 #------------------------ Meteor mask ---------------------------------
2482 #------------------------ Meteor mask ---------------------------------
2483 # #SNR mask
2483 # #SNR mask
2484 # boolMet = (SNRdB>SNRthresh)#|(~numpy.isnan(SNRdB))
2484 # boolMet = (SNRdB>SNRthresh)#|(~numpy.isnan(SNRdB))
2485 #
2485 #
2486 # #Erase small objects
2486 # #Erase small objects
2487 # boolMet1 = self.__erase_small(boolMet, 2*sec, 5)
2487 # boolMet1 = self.__erase_small(boolMet, 2*sec, 5)
2488 #
2488 #
2489 # auxEEJ = numpy.sum(boolMet1,axis=0)
2489 # auxEEJ = numpy.sum(boolMet1,axis=0)
2490 # indOver = auxEEJ>nProfiles*0.8 #Use this later
2490 # indOver = auxEEJ>nProfiles*0.8 #Use this later
2491 # indEEJ = numpy.where(indOver)[0]
2491 # indEEJ = numpy.where(indOver)[0]
2492 # indNEEJ = numpy.where(~indOver)[0]
2492 # indNEEJ = numpy.where(~indOver)[0]
2493 #
2493 #
2494 # boolMetFin = boolMet1
2494 # boolMetFin = boolMet1
2495 #
2495 #
2496 # if indEEJ.size > 0:
2496 # if indEEJ.size > 0:
2497 # boolMet1[:,indEEJ] = False #Erase heights with EEJ
2497 # boolMet1[:,indEEJ] = False #Erase heights with EEJ
2498 #
2498 #
2499 # boolMet2 = coh > cohThresh
2499 # boolMet2 = coh > cohThresh
2500 # boolMet2 = self.__erase_small(boolMet2, 2*sec,5)
2500 # boolMet2 = self.__erase_small(boolMet2, 2*sec,5)
2501 #
2501 #
2502 # #Final Meteor mask
2502 # #Final Meteor mask
2503 # boolMetFin = boolMet1|boolMet2
2503 # boolMetFin = boolMet1|boolMet2
2504
2504
2505 #Coherence mask
2505 #Coherence mask
2506 boolMet1 = coh > 0.75
2506 boolMet1 = coh > 0.75
2507 struc = numpy.ones((30,1))
2507 struc = numpy.ones((30,1))
2508 boolMet1 = ndimage.morphology.binary_dilation(boolMet1, structure=struc)
2508 boolMet1 = ndimage.morphology.binary_dilation(boolMet1, structure=struc)
2509
2509
2510 #Derivative mask
2510 #Derivative mask
2511 derPhase = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
2511 derPhase = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
2512 boolMet2 = derPhase < 0.2
2512 boolMet2 = derPhase < 0.2
2513 # boolMet2 = ndimage.morphology.binary_opening(boolMet2)
2513 # boolMet2 = ndimage.morphology.binary_opening(boolMet2)
2514 # boolMet2 = ndimage.morphology.binary_closing(boolMet2, structure = numpy.ones((10,1)))
2514 # boolMet2 = ndimage.morphology.binary_closing(boolMet2, structure = numpy.ones((10,1)))
2515 boolMet2 = ndimage.median_filter(boolMet2,size=5)
2515 boolMet2 = ndimage.median_filter(boolMet2,size=5)
2516 boolMet2 = numpy.vstack((boolMet2,numpy.full((1,nHeights), True, dtype=bool)))
2516 boolMet2 = numpy.vstack((boolMet2,numpy.full((1,nHeights), True, dtype=bool)))
2517 # #Final mask
2517 # #Final mask
2518 # boolMetFin = boolMet2
2518 # boolMetFin = boolMet2
2519 boolMetFin = boolMet1&boolMet2
2519 boolMetFin = boolMet1&boolMet2
2520 # boolMetFin = ndimage.morphology.binary_dilation(boolMetFin)
2520 # boolMetFin = ndimage.morphology.binary_dilation(boolMetFin)
2521 #Creating data_param
2521 #Creating data_param
2522 coordMet = numpy.where(boolMetFin)
2522 coordMet = numpy.where(boolMetFin)
2523
2523
2524 tmet = coordMet[0]
2524 tmet = coordMet[0]
2525 hmet = coordMet[1]
2525 hmet = coordMet[1]
2526
2526
2527 data_param = numpy.zeros((tmet.size, 6 + nPairs))
2527 data_param = numpy.zeros((tmet.size, 6 + nPairs))
2528 data_param[:,0] = utctime
2528 data_param[:,0] = utctime
2529 data_param[:,1] = tmet
2529 data_param[:,1] = tmet
2530 data_param[:,2] = hmet
2530 data_param[:,2] = hmet
2531 data_param[:,3] = SNRm[tmet,hmet]
2531 data_param[:,3] = SNRm[tmet,hmet]
2532 data_param[:,4] = velRad[tmet,hmet]
2532 data_param[:,4] = velRad[tmet,hmet]
2533 data_param[:,5] = coh[tmet,hmet]
2533 data_param[:,5] = coh[tmet,hmet]
2534 data_param[:,6:] = phase[:,tmet,hmet].T
2534 data_param[:,6:] = phase[:,tmet,hmet].T
2535
2535
2536 elif mode == 'DBS':
2536 elif mode == 'DBS':
2537 dataOut.groupList = numpy.arange(nChannels)
2537 dataOut.groupList = numpy.arange(nChannels)
2538
2538
2539 #Radial Velocities
2539 #Radial Velocities
2540 phase = numpy.angle(data_acf[:,1,:,:])
2540 phase = numpy.angle(data_acf[:,1,:,:])
2541 # phase = ndimage.median_filter(numpy.angle(data_acf[:,1,:,:]), size = (1,5,1))
2541 # phase = ndimage.median_filter(numpy.angle(data_acf[:,1,:,:]), size = (1,5,1))
2542 velRad = phase*lamb/(4*numpy.pi*tSamp)
2542 velRad = phase*lamb/(4*numpy.pi*tSamp)
2543
2543
2544 #Spectral width
2544 #Spectral width
2545 # acf1 = ndimage.median_filter(numpy.abs(data_acf[:,1,:,:]), size = (1,5,1))
2545 # acf1 = ndimage.median_filter(numpy.abs(data_acf[:,1,:,:]), size = (1,5,1))
2546 # acf2 = ndimage.median_filter(numpy.abs(data_acf[:,2,:,:]), size = (1,5,1))
2546 # acf2 = ndimage.median_filter(numpy.abs(data_acf[:,2,:,:]), size = (1,5,1))
2547 acf1 = data_acf[:,1,:,:]
2547 acf1 = data_acf[:,1,:,:]
2548 acf2 = data_acf[:,2,:,:]
2548 acf2 = data_acf[:,2,:,:]
2549
2549
2550 spcWidth = (lamb/(2*numpy.sqrt(6)*numpy.pi*tSamp))*numpy.sqrt(numpy.log(acf1/acf2))
2550 spcWidth = (lamb/(2*numpy.sqrt(6)*numpy.pi*tSamp))*numpy.sqrt(numpy.log(acf1/acf2))
2551 # velRad = ndimage.median_filter(velRad, size = (1,5,1))
2551 # velRad = ndimage.median_filter(velRad, size = (1,5,1))
2552 if allData:
2552 if allData:
2553 boolMetFin = ~numpy.isnan(SNRdB)
2553 boolMetFin = ~numpy.isnan(SNRdB)
2554 else:
2554 else:
2555 #SNR
2555 #SNR
2556 boolMet1 = (SNRdB>SNRthresh) #SNR mask
2556 boolMet1 = (SNRdB>SNRthresh) #SNR mask
2557 boolMet1 = ndimage.median_filter(boolMet1, size=(1,5,5))
2557 boolMet1 = ndimage.median_filter(boolMet1, size=(1,5,5))
2558
2558
2559 #Radial velocity
2559 #Radial velocity
2560 boolMet2 = numpy.abs(velRad) < 20
2560 boolMet2 = numpy.abs(velRad) < 20
2561 boolMet2 = ndimage.median_filter(boolMet2, (1,5,5))
2561 boolMet2 = ndimage.median_filter(boolMet2, (1,5,5))
2562
2562
2563 #Spectral Width
2563 #Spectral Width
2564 boolMet3 = spcWidth < 30
2564 boolMet3 = spcWidth < 30
2565 boolMet3 = ndimage.median_filter(boolMet3, (1,5,5))
2565 boolMet3 = ndimage.median_filter(boolMet3, (1,5,5))
2566 # boolMetFin = self.__erase_small(boolMet1, 10,5)
2566 # boolMetFin = self.__erase_small(boolMet1, 10,5)
2567 boolMetFin = boolMet1&boolMet2&boolMet3
2567 boolMetFin = boolMet1&boolMet2&boolMet3
2568
2568
2569 #Creating data_param
2569 #Creating data_param
2570 coordMet = numpy.where(boolMetFin)
2570 coordMet = numpy.where(boolMetFin)
2571
2571
2572 cmet = coordMet[0]
2572 cmet = coordMet[0]
2573 tmet = coordMet[1]
2573 tmet = coordMet[1]
2574 hmet = coordMet[2]
2574 hmet = coordMet[2]
2575
2575
2576 data_param = numpy.zeros((tmet.size, 7))
2576 data_param = numpy.zeros((tmet.size, 7))
2577 data_param[:,0] = utctime
2577 data_param[:,0] = utctime
2578 data_param[:,1] = cmet
2578 data_param[:,1] = cmet
2579 data_param[:,2] = tmet
2579 data_param[:,2] = tmet
2580 data_param[:,3] = hmet
2580 data_param[:,3] = hmet
2581 data_param[:,4] = SNR[cmet,tmet,hmet].T
2581 data_param[:,4] = SNR[cmet,tmet,hmet].T
2582 data_param[:,5] = velRad[cmet,tmet,hmet].T
2582 data_param[:,5] = velRad[cmet,tmet,hmet].T
2583 data_param[:,6] = spcWidth[cmet,tmet,hmet].T
2583 data_param[:,6] = spcWidth[cmet,tmet,hmet].T
2584
2584
2585 # self.dataOut.data_param = data_int
2585 # self.dataOut.data_param = data_int
2586 if len(data_param) == 0:
2586 if len(data_param) == 0:
2587 dataOut.flagNoData = True
2587 dataOut.flagNoData = True
2588 else:
2588 else:
2589 dataOut.data_param = data_param
2589 dataOut.data_param = data_param
2590
2590
2591 def __erase_small(self, binArray, threshX, threshY):
2591 def __erase_small(self, binArray, threshX, threshY):
2592 labarray, numfeat = ndimage.measurements.label(binArray)
2592 labarray, numfeat = ndimage.measurements.label(binArray)
2593 binArray1 = numpy.copy(binArray)
2593 binArray1 = numpy.copy(binArray)
2594
2594
2595 for i in range(1,numfeat + 1):
2595 for i in range(1,numfeat + 1):
2596 auxBin = (labarray==i)
2596 auxBin = (labarray==i)
2597 auxSize = auxBin.sum()
2597 auxSize = auxBin.sum()
2598
2598
2599 x,y = numpy.where(auxBin)
2599 x,y = numpy.where(auxBin)
2600 widthX = x.max() - x.min()
2600 widthX = x.max() - x.min()
2601 widthY = y.max() - y.min()
2601 widthY = y.max() - y.min()
2602
2602
2603 #width X: 3 seg -> 12.5*3
2603 #width X: 3 seg -> 12.5*3
2604 #width Y:
2604 #width Y:
2605
2605
2606 if (auxSize < 50) or (widthX < threshX) or (widthY < threshY):
2606 if (auxSize < 50) or (widthX < threshX) or (widthY < threshY):
2607 binArray1[auxBin] = False
2607 binArray1[auxBin] = False
2608
2608
2609 return binArray1
2609 return binArray1
2610
2610
2611 #--------------- Specular Meteor ----------------
2611 #--------------- Specular Meteor ----------------
2612
2612
2613 class SMDetection(Operation):
2613 class SMDetection(Operation):
2614 '''
2614 '''
2615 Function DetectMeteors()
2615 Function DetectMeteors()
2616 Project developed with paper:
2616 Project developed with paper:
2617 HOLDSWORTH ET AL. 2004
2617 HOLDSWORTH ET AL. 2004
2618
2618
2619 Input:
2619 Input:
2620 self.dataOut.data_pre
2620 self.dataOut.data_pre
2621
2621
2622 centerReceiverIndex: From the channels, which is the center receiver
2622 centerReceiverIndex: From the channels, which is the center receiver
2623
2623
2624 hei_ref: Height reference for the Beacon signal extraction
2624 hei_ref: Height reference for the Beacon signal extraction
2625 tauindex:
2625 tauindex:
2626 predefinedPhaseShifts: Predefined phase offset for the voltge signals
2626 predefinedPhaseShifts: Predefined phase offset for the voltge signals
2627
2627
2628 cohDetection: Whether to user Coherent detection or not
2628 cohDetection: Whether to user Coherent detection or not
2629 cohDet_timeStep: Coherent Detection calculation time step
2629 cohDet_timeStep: Coherent Detection calculation time step
2630 cohDet_thresh: Coherent Detection phase threshold to correct phases
2630 cohDet_thresh: Coherent Detection phase threshold to correct phases
2631
2631
2632 noise_timeStep: Noise calculation time step
2632 noise_timeStep: Noise calculation time step
2633 noise_multiple: Noise multiple to define signal threshold
2633 noise_multiple: Noise multiple to define signal threshold
2634
2634
2635 multDet_timeLimit: Multiple Detection Removal time limit in seconds
2635 multDet_timeLimit: Multiple Detection Removal time limit in seconds
2636 multDet_rangeLimit: Multiple Detection Removal range limit in km
2636 multDet_rangeLimit: Multiple Detection Removal range limit in km
2637
2637
2638 phaseThresh: Maximum phase difference between receiver to be consider a meteor
2638 phaseThresh: Maximum phase difference between receiver to be consider a meteor
2639 SNRThresh: Minimum SNR threshold of the meteor signal to be consider a meteor
2639 SNRThresh: Minimum SNR threshold of the meteor signal to be consider a meteor
2640
2640
2641 hmin: Minimum Height of the meteor to use it in the further wind estimations
2641 hmin: Minimum Height of the meteor to use it in the further wind estimations
2642 hmax: Maximum Height of the meteor to use it in the further wind estimations
2642 hmax: Maximum Height of the meteor to use it in the further wind estimations
2643 azimuth: Azimuth angle correction
2643 azimuth: Azimuth angle correction
2644
2644
2645 Affected:
2645 Affected:
2646 self.dataOut.data_param
2646 self.dataOut.data_param
2647
2647
2648 Rejection Criteria (Errors):
2648 Rejection Criteria (Errors):
2649 0: No error; analysis OK
2649 0: No error; analysis OK
2650 1: SNR < SNR threshold
2650 1: SNR < SNR threshold
2651 2: angle of arrival (AOA) ambiguously determined
2651 2: angle of arrival (AOA) ambiguously determined
2652 3: AOA estimate not feasible
2652 3: AOA estimate not feasible
2653 4: Large difference in AOAs obtained from different antenna baselines
2653 4: Large difference in AOAs obtained from different antenna baselines
2654 5: echo at start or end of time series
2654 5: echo at start or end of time series
2655 6: echo less than 5 examples long; too short for analysis
2655 6: echo less than 5 examples long; too short for analysis
2656 7: echo rise exceeds 0.3s
2656 7: echo rise exceeds 0.3s
2657 8: echo decay time less than twice rise time
2657 8: echo decay time less than twice rise time
2658 9: large power level before echo
2658 9: large power level before echo
2659 10: large power level after echo
2659 10: large power level after echo
2660 11: poor fit to amplitude for estimation of decay time
2660 11: poor fit to amplitude for estimation of decay time
2661 12: poor fit to CCF phase variation for estimation of radial drift velocity
2661 12: poor fit to CCF phase variation for estimation of radial drift velocity
2662 13: height unresolvable echo: not valid height within 70 to 110 km
2662 13: height unresolvable echo: not valid height within 70 to 110 km
2663 14: height ambiguous echo: more then one possible height within 70 to 110 km
2663 14: height ambiguous echo: more then one possible height within 70 to 110 km
2664 15: radial drift velocity or projected horizontal velocity exceeds 200 m/s
2664 15: radial drift velocity or projected horizontal velocity exceeds 200 m/s
2665 16: oscilatory echo, indicating event most likely not an underdense echo
2665 16: oscilatory echo, indicating event most likely not an underdense echo
2666
2666
2667 17: phase difference in meteor Reestimation
2667 17: phase difference in meteor Reestimation
2668
2668
2669 Data Storage:
2669 Data Storage:
2670 Meteors for Wind Estimation (8):
2670 Meteors for Wind Estimation (8):
2671 Utc Time | Range Height
2671 Utc Time | Range Height
2672 Azimuth Zenith errorCosDir
2672 Azimuth Zenith errorCosDir
2673 VelRad errorVelRad
2673 VelRad errorVelRad
2674 Phase0 Phase1 Phase2 Phase3
2674 Phase0 Phase1 Phase2 Phase3
2675 TypeError
2675 TypeError
2676
2676
2677 '''
2677 '''
2678
2678
2679 def run(self, dataOut, hei_ref = None, tauindex = 0,
2679 def run(self, dataOut, hei_ref = None, tauindex = 0,
2680 phaseOffsets = None,
2680 phaseOffsets = None,
2681 cohDetection = False, cohDet_timeStep = 1, cohDet_thresh = 25,
2681 cohDetection = False, cohDet_timeStep = 1, cohDet_thresh = 25,
2682 noise_timeStep = 4, noise_multiple = 4,
2682 noise_timeStep = 4, noise_multiple = 4,
2683 multDet_timeLimit = 1, multDet_rangeLimit = 3,
2683 multDet_timeLimit = 1, multDet_rangeLimit = 3,
2684 phaseThresh = 20, SNRThresh = 5,
2684 phaseThresh = 20, SNRThresh = 5,
2685 hmin = 50, hmax=150, azimuth = 0,
2685 hmin = 50, hmax=150, azimuth = 0,
2686 channelPositions = None) :
2686 channelPositions = None) :
2687
2687
2688
2688
2689 #Getting Pairslist
2689 #Getting Pairslist
2690 if channelPositions is None:
2690 if channelPositions is None:
2691 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
2691 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
2692 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
2692 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
2693 meteorOps = SMOperations()
2693 meteorOps = SMOperations()
2694 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
2694 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
2695 heiRang = dataOut.heightList
2695 heiRang = dataOut.heightList
2696 #Get Beacon signal - No Beacon signal anymore
2696 #Get Beacon signal - No Beacon signal anymore
2697 # newheis = numpy.where(self.dataOut.heightList>self.dataOut.radarControllerHeaderObj.Taus[tauindex])
2697 # newheis = numpy.where(self.dataOut.heightList>self.dataOut.radarControllerHeaderObj.Taus[tauindex])
2698 #
2698 #
2699 # if hei_ref != None:
2699 # if hei_ref != None:
2700 # newheis = numpy.where(self.dataOut.heightList>hei_ref)
2700 # newheis = numpy.where(self.dataOut.heightList>hei_ref)
2701 #
2701 #
2702
2702
2703
2703
2704 #****************REMOVING HARDWARE PHASE DIFFERENCES***************
2704 #****************REMOVING HARDWARE PHASE DIFFERENCES***************
2705 # see if the user put in pre defined phase shifts
2705 # see if the user put in pre defined phase shifts
2706 voltsPShift = dataOut.data_pre.copy()
2706 voltsPShift = dataOut.data_pre.copy()
2707
2707
2708 # if predefinedPhaseShifts != None:
2708 # if predefinedPhaseShifts != None:
2709 # hardwarePhaseShifts = numpy.array(predefinedPhaseShifts)*numpy.pi/180
2709 # hardwarePhaseShifts = numpy.array(predefinedPhaseShifts)*numpy.pi/180
2710 #
2710 #
2711 # # elif beaconPhaseShifts:
2711 # # elif beaconPhaseShifts:
2712 # # #get hardware phase shifts using beacon signal
2712 # # #get hardware phase shifts using beacon signal
2713 # # hardwarePhaseShifts = self.__getHardwarePhaseDiff(self.dataOut.data_pre, pairslist, newheis, 10)
2713 # # hardwarePhaseShifts = self.__getHardwarePhaseDiff(self.dataOut.data_pre, pairslist, newheis, 10)
2714 # # hardwarePhaseShifts = numpy.insert(hardwarePhaseShifts,centerReceiverIndex,0)
2714 # # hardwarePhaseShifts = numpy.insert(hardwarePhaseShifts,centerReceiverIndex,0)
2715 #
2715 #
2716 # else:
2716 # else:
2717 # hardwarePhaseShifts = numpy.zeros(5)
2717 # hardwarePhaseShifts = numpy.zeros(5)
2718 #
2718 #
2719 # voltsPShift = numpy.zeros((self.dataOut.data_pre.shape[0],self.dataOut.data_pre.shape[1],self.dataOut.data_pre.shape[2]), dtype = 'complex')
2719 # voltsPShift = numpy.zeros((self.dataOut.data_pre.shape[0],self.dataOut.data_pre.shape[1],self.dataOut.data_pre.shape[2]), dtype = 'complex')
2720 # for i in range(self.dataOut.data_pre.shape[0]):
2720 # for i in range(self.dataOut.data_pre.shape[0]):
2721 # voltsPShift[i,:,:] = self.__shiftPhase(self.dataOut.data_pre[i,:,:], hardwarePhaseShifts[i])
2721 # voltsPShift[i,:,:] = self.__shiftPhase(self.dataOut.data_pre[i,:,:], hardwarePhaseShifts[i])
2722
2722
2723 #******************END OF REMOVING HARDWARE PHASE DIFFERENCES*********
2723 #******************END OF REMOVING HARDWARE PHASE DIFFERENCES*********
2724
2724
2725 #Remove DC
2725 #Remove DC
2726 voltsDC = numpy.mean(voltsPShift,1)
2726 voltsDC = numpy.mean(voltsPShift,1)
2727 voltsDC = numpy.mean(voltsDC,1)
2727 voltsDC = numpy.mean(voltsDC,1)
2728 for i in range(voltsDC.shape[0]):
2728 for i in range(voltsDC.shape[0]):
2729 voltsPShift[i] = voltsPShift[i] - voltsDC[i]
2729 voltsPShift[i] = voltsPShift[i] - voltsDC[i]
2730
2730
2731 #Don't considerate last heights, theyre used to calculate Hardware Phase Shift
2731 #Don't considerate last heights, theyre used to calculate Hardware Phase Shift
2732 # voltsPShift = voltsPShift[:,:,:newheis[0][0]]
2732 # voltsPShift = voltsPShift[:,:,:newheis[0][0]]
2733
2733
2734 #************ FIND POWER OF DATA W/COH OR NON COH DETECTION (3.4) **********
2734 #************ FIND POWER OF DATA W/COH OR NON COH DETECTION (3.4) **********
2735 #Coherent Detection
2735 #Coherent Detection
2736 if cohDetection:
2736 if cohDetection:
2737 #use coherent detection to get the net power
2737 #use coherent detection to get the net power
2738 cohDet_thresh = cohDet_thresh*numpy.pi/180
2738 cohDet_thresh = cohDet_thresh*numpy.pi/180
2739 voltsPShift = self.__coherentDetection(voltsPShift, cohDet_timeStep, dataOut.timeInterval, pairslist0, cohDet_thresh)
2739 voltsPShift = self.__coherentDetection(voltsPShift, cohDet_timeStep, dataOut.timeInterval, pairslist0, cohDet_thresh)
2740
2740
2741 #Non-coherent detection!
2741 #Non-coherent detection!
2742 powerNet = numpy.nansum(numpy.abs(voltsPShift[:,:,:])**2,0)
2742 powerNet = numpy.nansum(numpy.abs(voltsPShift[:,:,:])**2,0)
2743 #********** END OF COH/NON-COH POWER CALCULATION**********************
2743 #********** END OF COH/NON-COH POWER CALCULATION**********************
2744
2744
2745 #********** FIND THE NOISE LEVEL AND POSSIBLE METEORS ****************
2745 #********** FIND THE NOISE LEVEL AND POSSIBLE METEORS ****************
2746 #Get noise
2746 #Get noise
2747 noise, noise1 = self.__getNoise(powerNet, noise_timeStep, dataOut.timeInterval)
2747 noise, noise1 = self.__getNoise(powerNet, noise_timeStep, dataOut.timeInterval)
2748 # noise = self.getNoise1(powerNet, noise_timeStep, self.dataOut.timeInterval)
2748 # noise = self.getNoise1(powerNet, noise_timeStep, self.dataOut.timeInterval)
2749 #Get signal threshold
2749 #Get signal threshold
2750 signalThresh = noise_multiple*noise
2750 signalThresh = noise_multiple*noise
2751 #Meteor echoes detection
2751 #Meteor echoes detection
2752 listMeteors = self.__findMeteors(powerNet, signalThresh)
2752 listMeteors = self.__findMeteors(powerNet, signalThresh)
2753 #******* END OF NOISE LEVEL AND POSSIBLE METEORS CACULATION **********
2753 #******* END OF NOISE LEVEL AND POSSIBLE METEORS CACULATION **********
2754
2754
2755 #************** REMOVE MULTIPLE DETECTIONS (3.5) ***************************
2755 #************** REMOVE MULTIPLE DETECTIONS (3.5) ***************************
2756 #Parameters
2756 #Parameters
2757 heiRange = dataOut.heightList
2757 heiRange = dataOut.heightList
2758 rangeInterval = heiRange[1] - heiRange[0]
2758 rangeInterval = heiRange[1] - heiRange[0]
2759 rangeLimit = multDet_rangeLimit/rangeInterval
2759 rangeLimit = multDet_rangeLimit/rangeInterval
2760 timeLimit = multDet_timeLimit/dataOut.timeInterval
2760 timeLimit = multDet_timeLimit/dataOut.timeInterval
2761 #Multiple detection removals
2761 #Multiple detection removals
2762 listMeteors1 = self.__removeMultipleDetections(listMeteors, rangeLimit, timeLimit)
2762 listMeteors1 = self.__removeMultipleDetections(listMeteors, rangeLimit, timeLimit)
2763 #************ END OF REMOVE MULTIPLE DETECTIONS **********************
2763 #************ END OF REMOVE MULTIPLE DETECTIONS **********************
2764
2764
2765 #********************* METEOR REESTIMATION (3.7, 3.8, 3.9, 3.10) ********************
2765 #********************* METEOR REESTIMATION (3.7, 3.8, 3.9, 3.10) ********************
2766 #Parameters
2766 #Parameters
2767 phaseThresh = phaseThresh*numpy.pi/180
2767 phaseThresh = phaseThresh*numpy.pi/180
2768 thresh = [phaseThresh, noise_multiple, SNRThresh]
2768 thresh = [phaseThresh, noise_multiple, SNRThresh]
2769 #Meteor reestimation (Errors N 1, 6, 12, 17)
2769 #Meteor reestimation (Errors N 1, 6, 12, 17)
2770 listMeteors2, listMeteorsPower, listMeteorsVolts = self.__meteorReestimation(listMeteors1, voltsPShift, pairslist0, thresh, noise, dataOut.timeInterval, dataOut.frequency)
2770 listMeteors2, listMeteorsPower, listMeteorsVolts = self.__meteorReestimation(listMeteors1, voltsPShift, pairslist0, thresh, noise, dataOut.timeInterval, dataOut.frequency)
2771 # listMeteors2, listMeteorsPower, listMeteorsVolts = self.meteorReestimation3(listMeteors2, listMeteorsPower, listMeteorsVolts, voltsPShift, pairslist, thresh, noise)
2771 # listMeteors2, listMeteorsPower, listMeteorsVolts = self.meteorReestimation3(listMeteors2, listMeteorsPower, listMeteorsVolts, voltsPShift, pairslist, thresh, noise)
2772 #Estimation of decay times (Errors N 7, 8, 11)
2772 #Estimation of decay times (Errors N 7, 8, 11)
2773 listMeteors3 = self.__estimateDecayTime(listMeteors2, listMeteorsPower, dataOut.timeInterval, dataOut.frequency)
2773 listMeteors3 = self.__estimateDecayTime(listMeteors2, listMeteorsPower, dataOut.timeInterval, dataOut.frequency)
2774 #******************* END OF METEOR REESTIMATION *******************
2774 #******************* END OF METEOR REESTIMATION *******************
2775
2775
2776 #********************* METEOR PARAMETERS CALCULATION (3.11, 3.12, 3.13) **************************
2776 #********************* METEOR PARAMETERS CALCULATION (3.11, 3.12, 3.13) **************************
2777 #Calculating Radial Velocity (Error N 15)
2777 #Calculating Radial Velocity (Error N 15)
2778 radialStdThresh = 10
2778 radialStdThresh = 10
2779 listMeteors4 = self.__getRadialVelocity(listMeteors3, listMeteorsVolts, radialStdThresh, pairslist0, dataOut.timeInterval)
2779 listMeteors4 = self.__getRadialVelocity(listMeteors3, listMeteorsVolts, radialStdThresh, pairslist0, dataOut.timeInterval)
2780
2780
2781 if len(listMeteors4) > 0:
2781 if len(listMeteors4) > 0:
2782 #Setting New Array
2782 #Setting New Array
2783 date = dataOut.utctime
2783 date = dataOut.utctime
2784 arrayParameters = self.__setNewArrays(listMeteors4, date, heiRang)
2784 arrayParameters = self.__setNewArrays(listMeteors4, date, heiRang)
2785
2785
2786 #Correcting phase offset
2786 #Correcting phase offset
2787 if phaseOffsets != None:
2787 if phaseOffsets != None:
2788 phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
2788 phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
2789 arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
2789 arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
2790
2790
2791 #Second Pairslist
2791 #Second Pairslist
2792 pairsList = []
2792 pairsList = []
2793 pairx = (0,1)
2793 pairx = (0,1)
2794 pairy = (2,3)
2794 pairy = (2,3)
2795 pairsList.append(pairx)
2795 pairsList.append(pairx)
2796 pairsList.append(pairy)
2796 pairsList.append(pairy)
2797
2797
2798 jph = numpy.array([0,0,0,0])
2798 jph = numpy.array([0,0,0,0])
2799 h = (hmin,hmax)
2799 h = (hmin,hmax)
2800 arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
2800 arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
2801
2801
2802 # #Calculate AOA (Error N 3, 4)
2802 # #Calculate AOA (Error N 3, 4)
2803 # #JONES ET AL. 1998
2803 # #JONES ET AL. 1998
2804 # error = arrayParameters[:,-1]
2804 # error = arrayParameters[:,-1]
2805 # AOAthresh = numpy.pi/8
2805 # AOAthresh = numpy.pi/8
2806 # phases = -arrayParameters[:,9:13]
2806 # phases = -arrayParameters[:,9:13]
2807 # arrayParameters[:,4:7], arrayParameters[:,-1] = meteorOps.getAOA(phases, pairsList, error, AOAthresh, azimuth)
2807 # arrayParameters[:,4:7], arrayParameters[:,-1] = meteorOps.getAOA(phases, pairsList, error, AOAthresh, azimuth)
2808 #
2808 #
2809 # #Calculate Heights (Error N 13 and 14)
2809 # #Calculate Heights (Error N 13 and 14)
2810 # error = arrayParameters[:,-1]
2810 # error = arrayParameters[:,-1]
2811 # Ranges = arrayParameters[:,2]
2811 # Ranges = arrayParameters[:,2]
2812 # zenith = arrayParameters[:,5]
2812 # zenith = arrayParameters[:,5]
2813 # arrayParameters[:,3], arrayParameters[:,-1] = meteorOps.getHeights(Ranges, zenith, error, hmin, hmax)
2813 # arrayParameters[:,3], arrayParameters[:,-1] = meteorOps.getHeights(Ranges, zenith, error, hmin, hmax)
2814 # error = arrayParameters[:,-1]
2814 # error = arrayParameters[:,-1]
2815 #********************* END OF PARAMETERS CALCULATION **************************
2815 #********************* END OF PARAMETERS CALCULATION **************************
2816
2816
2817 #***************************+ PASS DATA TO NEXT STEP **********************
2817 #***************************+ PASS DATA TO NEXT STEP **********************
2818 # arrayFinal = arrayParameters.reshape((1,arrayParameters.shape[0],arrayParameters.shape[1]))
2818 # arrayFinal = arrayParameters.reshape((1,arrayParameters.shape[0],arrayParameters.shape[1]))
2819 dataOut.data_param = arrayParameters
2819 dataOut.data_param = arrayParameters
2820
2820
2821 if arrayParameters is None:
2821 if arrayParameters is None:
2822 dataOut.flagNoData = True
2822 dataOut.flagNoData = True
2823 else:
2823 else:
2824 dataOut.flagNoData = True
2824 dataOut.flagNoData = True
2825
2825
2826 return
2826 return
2827
2827
2828 def __getHardwarePhaseDiff(self, voltage0, pairslist, newheis, n):
2828 def __getHardwarePhaseDiff(self, voltage0, pairslist, newheis, n):
2829
2829
2830 minIndex = min(newheis[0])
2830 minIndex = min(newheis[0])
2831 maxIndex = max(newheis[0])
2831 maxIndex = max(newheis[0])
2832
2832
2833 voltage = voltage0[:,:,minIndex:maxIndex+1]
2833 voltage = voltage0[:,:,minIndex:maxIndex+1]
2834 nLength = voltage.shape[1]/n
2834 nLength = voltage.shape[1]/n
2835 nMin = 0
2835 nMin = 0
2836 nMax = 0
2836 nMax = 0
2837 phaseOffset = numpy.zeros((len(pairslist),n))
2837 phaseOffset = numpy.zeros((len(pairslist),n))
2838
2838
2839 for i in range(n):
2839 for i in range(n):
2840 nMax += nLength
2840 nMax += nLength
2841 phaseCCF = -numpy.angle(self.__calculateCCF(voltage[:,nMin:nMax,:], pairslist, [0]))
2841 phaseCCF = -numpy.angle(self.__calculateCCF(voltage[:,nMin:nMax,:], pairslist, [0]))
2842 phaseCCF = numpy.mean(phaseCCF, axis = 2)
2842 phaseCCF = numpy.mean(phaseCCF, axis = 2)
2843 phaseOffset[:,i] = phaseCCF.transpose()
2843 phaseOffset[:,i] = phaseCCF.transpose()
2844 nMin = nMax
2844 nMin = nMax
2845 # phaseDiff, phaseArrival = self.estimatePhaseDifference(voltage, pairslist)
2845 # phaseDiff, phaseArrival = self.estimatePhaseDifference(voltage, pairslist)
2846
2846
2847 #Remove Outliers
2847 #Remove Outliers
2848 factor = 2
2848 factor = 2
2849 wt = phaseOffset - signal.medfilt(phaseOffset,(1,5))
2849 wt = phaseOffset - signal.medfilt(phaseOffset,(1,5))
2850 dw = numpy.std(wt,axis = 1)
2850 dw = numpy.std(wt,axis = 1)
2851 dw = dw.reshape((dw.size,1))
2851 dw = dw.reshape((dw.size,1))
2852 ind = numpy.where(numpy.logical_or(wt>dw*factor,wt<-dw*factor))
2852 ind = numpy.where(numpy.logical_or(wt>dw*factor,wt<-dw*factor))
2853 phaseOffset[ind] = numpy.nan
2853 phaseOffset[ind] = numpy.nan
2854 phaseOffset = stats.nanmean(phaseOffset, axis=1)
2854 phaseOffset = stats.nanmean(phaseOffset, axis=1)
2855
2855
2856 return phaseOffset
2856 return phaseOffset
2857
2857
2858 def __shiftPhase(self, data, phaseShift):
2858 def __shiftPhase(self, data, phaseShift):
2859 #this will shift the phase of a complex number
2859 #this will shift the phase of a complex number
2860 dataShifted = numpy.abs(data) * numpy.exp((numpy.angle(data)+phaseShift)*1j)
2860 dataShifted = numpy.abs(data) * numpy.exp((numpy.angle(data)+phaseShift)*1j)
2861 return dataShifted
2861 return dataShifted
2862
2862
2863 def __estimatePhaseDifference(self, array, pairslist):
2863 def __estimatePhaseDifference(self, array, pairslist):
2864 nChannel = array.shape[0]
2864 nChannel = array.shape[0]
2865 nHeights = array.shape[2]
2865 nHeights = array.shape[2]
2866 numPairs = len(pairslist)
2866 numPairs = len(pairslist)
2867 # phaseCCF = numpy.zeros((nChannel, 5, nHeights))
2867 # phaseCCF = numpy.zeros((nChannel, 5, nHeights))
2868 phaseCCF = numpy.angle(self.__calculateCCF(array, pairslist, [-2,-1,0,1,2]))
2868 phaseCCF = numpy.angle(self.__calculateCCF(array, pairslist, [-2,-1,0,1,2]))
2869
2869
2870 #Correct phases
2870 #Correct phases
2871 derPhaseCCF = phaseCCF[:,1:,:] - phaseCCF[:,0:-1,:]
2871 derPhaseCCF = phaseCCF[:,1:,:] - phaseCCF[:,0:-1,:]
2872 indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
2872 indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
2873
2873
2874 if indDer[0].shape[0] > 0:
2874 if indDer[0].shape[0] > 0:
2875 for i in range(indDer[0].shape[0]):
2875 for i in range(indDer[0].shape[0]):
2876 signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i],indDer[2][i]])
2876 signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i],indDer[2][i]])
2877 phaseCCF[indDer[0][i],indDer[1][i]+1:,:] += signo*2*numpy.pi
2877 phaseCCF[indDer[0][i],indDer[1][i]+1:,:] += signo*2*numpy.pi
2878
2878
2879 # for j in range(numSides):
2879 # for j in range(numSides):
2880 # phaseCCFAux = self.calculateCCF(arrayCenter, arraySides[j,:,:], [-2,1,0,1,2])
2880 # phaseCCFAux = self.calculateCCF(arrayCenter, arraySides[j,:,:], [-2,1,0,1,2])
2881 # phaseCCF[j,:,:] = numpy.angle(phaseCCFAux)
2881 # phaseCCF[j,:,:] = numpy.angle(phaseCCFAux)
2882 #
2882 #
2883 #Linear
2883 #Linear
2884 phaseInt = numpy.zeros((numPairs,1))
2884 phaseInt = numpy.zeros((numPairs,1))
2885 angAllCCF = phaseCCF[:,[0,1,3,4],0]
2885 angAllCCF = phaseCCF[:,[0,1,3,4],0]
2886 for j in range(numPairs):
2886 for j in range(numPairs):
2887 fit = stats.linregress([-2,-1,1,2],angAllCCF[j,:])
2887 fit = stats.linregress([-2,-1,1,2],angAllCCF[j,:])
2888 phaseInt[j] = fit[1]
2888 phaseInt[j] = fit[1]
2889 #Phase Differences
2889 #Phase Differences
2890 phaseDiff = phaseInt - phaseCCF[:,2,:]
2890 phaseDiff = phaseInt - phaseCCF[:,2,:]
2891 phaseArrival = phaseInt.reshape(phaseInt.size)
2891 phaseArrival = phaseInt.reshape(phaseInt.size)
2892
2892
2893 #Dealias
2893 #Dealias
2894 phaseArrival = numpy.angle(numpy.exp(1j*phaseArrival))
2894 phaseArrival = numpy.angle(numpy.exp(1j*phaseArrival))
2895 # indAlias = numpy.where(phaseArrival > numpy.pi)
2895 # indAlias = numpy.where(phaseArrival > numpy.pi)
2896 # phaseArrival[indAlias] -= 2*numpy.pi
2896 # phaseArrival[indAlias] -= 2*numpy.pi
2897 # indAlias = numpy.where(phaseArrival < -numpy.pi)
2897 # indAlias = numpy.where(phaseArrival < -numpy.pi)
2898 # phaseArrival[indAlias] += 2*numpy.pi
2898 # phaseArrival[indAlias] += 2*numpy.pi
2899
2899
2900 return phaseDiff, phaseArrival
2900 return phaseDiff, phaseArrival
2901
2901
2902 def __coherentDetection(self, volts, timeSegment, timeInterval, pairslist, thresh):
2902 def __coherentDetection(self, volts, timeSegment, timeInterval, pairslist, thresh):
2903 #this function will run the coherent detection used in Holdworth et al. 2004 and return the net power
2903 #this function will run the coherent detection used in Holdworth et al. 2004 and return the net power
2904 #find the phase shifts of each channel over 1 second intervals
2904 #find the phase shifts of each channel over 1 second intervals
2905 #only look at ranges below the beacon signal
2905 #only look at ranges below the beacon signal
2906 numProfPerBlock = numpy.ceil(timeSegment/timeInterval)
2906 numProfPerBlock = numpy.ceil(timeSegment/timeInterval)
2907 numBlocks = int(volts.shape[1]/numProfPerBlock)
2907 numBlocks = int(volts.shape[1]/numProfPerBlock)
2908 numHeights = volts.shape[2]
2908 numHeights = volts.shape[2]
2909 nChannel = volts.shape[0]
2909 nChannel = volts.shape[0]
2910 voltsCohDet = volts.copy()
2910 voltsCohDet = volts.copy()
2911
2911
2912 pairsarray = numpy.array(pairslist)
2912 pairsarray = numpy.array(pairslist)
2913 indSides = pairsarray[:,1]
2913 indSides = pairsarray[:,1]
2914 # indSides = numpy.array(range(nChannel))
2914 # indSides = numpy.array(range(nChannel))
2915 # indSides = numpy.delete(indSides, indCenter)
2915 # indSides = numpy.delete(indSides, indCenter)
2916 #
2916 #
2917 # listCenter = numpy.array_split(volts[indCenter,:,:], numBlocks, 0)
2917 # listCenter = numpy.array_split(volts[indCenter,:,:], numBlocks, 0)
2918 listBlocks = numpy.array_split(volts, numBlocks, 1)
2918 listBlocks = numpy.array_split(volts, numBlocks, 1)
2919
2919
2920 startInd = 0
2920 startInd = 0
2921 endInd = 0
2921 endInd = 0
2922
2922
2923 for i in range(numBlocks):
2923 for i in range(numBlocks):
2924 startInd = endInd
2924 startInd = endInd
2925 endInd = endInd + listBlocks[i].shape[1]
2925 endInd = endInd + listBlocks[i].shape[1]
2926
2926
2927 arrayBlock = listBlocks[i]
2927 arrayBlock = listBlocks[i]
2928 # arrayBlockCenter = listCenter[i]
2928 # arrayBlockCenter = listCenter[i]
2929
2929
2930 #Estimate the Phase Difference
2930 #Estimate the Phase Difference
2931 phaseDiff, aux = self.__estimatePhaseDifference(arrayBlock, pairslist)
2931 phaseDiff, aux = self.__estimatePhaseDifference(arrayBlock, pairslist)
2932 #Phase Difference RMS
2932 #Phase Difference RMS
2933 arrayPhaseRMS = numpy.abs(phaseDiff)
2933 arrayPhaseRMS = numpy.abs(phaseDiff)
2934 phaseRMSaux = numpy.sum(arrayPhaseRMS < thresh,0)
2934 phaseRMSaux = numpy.sum(arrayPhaseRMS < thresh,0)
2935 indPhase = numpy.where(phaseRMSaux==4)
2935 indPhase = numpy.where(phaseRMSaux==4)
2936 #Shifting
2936 #Shifting
2937 if indPhase[0].shape[0] > 0:
2937 if indPhase[0].shape[0] > 0:
2938 for j in range(indSides.size):
2938 for j in range(indSides.size):
2939 arrayBlock[indSides[j],:,indPhase] = self.__shiftPhase(arrayBlock[indSides[j],:,indPhase], phaseDiff[j,indPhase].transpose())
2939 arrayBlock[indSides[j],:,indPhase] = self.__shiftPhase(arrayBlock[indSides[j],:,indPhase], phaseDiff[j,indPhase].transpose())
2940 voltsCohDet[:,startInd:endInd,:] = arrayBlock
2940 voltsCohDet[:,startInd:endInd,:] = arrayBlock
2941
2941
2942 return voltsCohDet
2942 return voltsCohDet
2943
2943
2944 def __calculateCCF(self, volts, pairslist ,laglist):
2944 def __calculateCCF(self, volts, pairslist ,laglist):
2945
2945
2946 nHeights = volts.shape[2]
2946 nHeights = volts.shape[2]
2947 nPoints = volts.shape[1]
2947 nPoints = volts.shape[1]
2948 voltsCCF = numpy.zeros((len(pairslist), len(laglist), nHeights),dtype = 'complex')
2948 voltsCCF = numpy.zeros((len(pairslist), len(laglist), nHeights),dtype = 'complex')
2949
2949
2950 for i in range(len(pairslist)):
2950 for i in range(len(pairslist)):
2951 volts1 = volts[pairslist[i][0]]
2951 volts1 = volts[pairslist[i][0]]
2952 volts2 = volts[pairslist[i][1]]
2952 volts2 = volts[pairslist[i][1]]
2953
2953
2954 for t in range(len(laglist)):
2954 for t in range(len(laglist)):
2955 idxT = laglist[t]
2955 idxT = laglist[t]
2956 if idxT >= 0:
2956 if idxT >= 0:
2957 vStacked = numpy.vstack((volts2[idxT:,:],
2957 vStacked = numpy.vstack((volts2[idxT:,:],
2958 numpy.zeros((idxT, nHeights),dtype='complex')))
2958 numpy.zeros((idxT, nHeights),dtype='complex')))
2959 else:
2959 else:
2960 vStacked = numpy.vstack((numpy.zeros((-idxT, nHeights),dtype='complex'),
2960 vStacked = numpy.vstack((numpy.zeros((-idxT, nHeights),dtype='complex'),
2961 volts2[:(nPoints + idxT),:]))
2961 volts2[:(nPoints + idxT),:]))
2962 voltsCCF[i,t,:] = numpy.sum((numpy.conjugate(volts1)*vStacked),axis=0)
2962 voltsCCF[i,t,:] = numpy.sum((numpy.conjugate(volts1)*vStacked),axis=0)
2963
2963
2964 vStacked = None
2964 vStacked = None
2965 return voltsCCF
2965 return voltsCCF
2966
2966
2967 def __getNoise(self, power, timeSegment, timeInterval):
2967 def __getNoise(self, power, timeSegment, timeInterval):
2968 numProfPerBlock = numpy.ceil(timeSegment/timeInterval)
2968 numProfPerBlock = numpy.ceil(timeSegment/timeInterval)
2969 numBlocks = int(power.shape[0]/numProfPerBlock)
2969 numBlocks = int(power.shape[0]/numProfPerBlock)
2970 numHeights = power.shape[1]
2970 numHeights = power.shape[1]
2971
2971
2972 listPower = numpy.array_split(power, numBlocks, 0)
2972 listPower = numpy.array_split(power, numBlocks, 0)
2973 noise = numpy.zeros((power.shape[0], power.shape[1]))
2973 noise = numpy.zeros((power.shape[0], power.shape[1]))
2974 noise1 = numpy.zeros((power.shape[0], power.shape[1]))
2974 noise1 = numpy.zeros((power.shape[0], power.shape[1]))
2975
2975
2976 startInd = 0
2976 startInd = 0
2977 endInd = 0
2977 endInd = 0
2978
2978
2979 for i in range(numBlocks): #split por canal
2979 for i in range(numBlocks): #split por canal
2980 startInd = endInd
2980 startInd = endInd
2981 endInd = endInd + listPower[i].shape[0]
2981 endInd = endInd + listPower[i].shape[0]
2982
2982
2983 arrayBlock = listPower[i]
2983 arrayBlock = listPower[i]
2984 noiseAux = numpy.mean(arrayBlock, 0)
2984 noiseAux = numpy.mean(arrayBlock, 0)
2985 # noiseAux = numpy.median(noiseAux)
2985 # noiseAux = numpy.median(noiseAux)
2986 # noiseAux = numpy.mean(arrayBlock)
2986 # noiseAux = numpy.mean(arrayBlock)
2987 noise[startInd:endInd,:] = noise[startInd:endInd,:] + noiseAux
2987 noise[startInd:endInd,:] = noise[startInd:endInd,:] + noiseAux
2988
2988
2989 noiseAux1 = numpy.mean(arrayBlock)
2989 noiseAux1 = numpy.mean(arrayBlock)
2990 noise1[startInd:endInd,:] = noise1[startInd:endInd,:] + noiseAux1
2990 noise1[startInd:endInd,:] = noise1[startInd:endInd,:] + noiseAux1
2991
2991
2992 return noise, noise1
2992 return noise, noise1
2993
2993
2994 def __findMeteors(self, power, thresh):
2994 def __findMeteors(self, power, thresh):
2995 nProf = power.shape[0]
2995 nProf = power.shape[0]
2996 nHeights = power.shape[1]
2996 nHeights = power.shape[1]
2997 listMeteors = []
2997 listMeteors = []
2998
2998
2999 for i in range(nHeights):
2999 for i in range(nHeights):
3000 powerAux = power[:,i]
3000 powerAux = power[:,i]
3001 threshAux = thresh[:,i]
3001 threshAux = thresh[:,i]
3002
3002
3003 indUPthresh = numpy.where(powerAux > threshAux)[0]
3003 indUPthresh = numpy.where(powerAux > threshAux)[0]
3004 indDNthresh = numpy.where(powerAux <= threshAux)[0]
3004 indDNthresh = numpy.where(powerAux <= threshAux)[0]
3005
3005
3006 j = 0
3006 j = 0
3007
3007
3008 while (j < indUPthresh.size - 2):
3008 while (j < indUPthresh.size - 2):
3009 if (indUPthresh[j + 2] == indUPthresh[j] + 2):
3009 if (indUPthresh[j + 2] == indUPthresh[j] + 2):
3010 indDNAux = numpy.where(indDNthresh > indUPthresh[j])
3010 indDNAux = numpy.where(indDNthresh > indUPthresh[j])
3011 indDNthresh = indDNthresh[indDNAux]
3011 indDNthresh = indDNthresh[indDNAux]
3012
3012
3013 if (indDNthresh.size > 0):
3013 if (indDNthresh.size > 0):
3014 indEnd = indDNthresh[0] - 1
3014 indEnd = indDNthresh[0] - 1
3015 indInit = indUPthresh[j]
3015 indInit = indUPthresh[j]
3016
3016
3017 meteor = powerAux[indInit:indEnd + 1]
3017 meteor = powerAux[indInit:indEnd + 1]
3018 indPeak = meteor.argmax() + indInit
3018 indPeak = meteor.argmax() + indInit
3019 FLA = sum(numpy.conj(meteor)*numpy.hstack((meteor[1:],0)))
3019 FLA = sum(numpy.conj(meteor)*numpy.hstack((meteor[1:],0)))
3020
3020
3021 listMeteors.append(numpy.array([i,indInit,indPeak,indEnd,FLA])) #CHEQUEAR!!!!!
3021 listMeteors.append(numpy.array([i,indInit,indPeak,indEnd,FLA])) #CHEQUEAR!!!!!
3022 j = numpy.where(indUPthresh == indEnd)[0] + 1
3022 j = numpy.where(indUPthresh == indEnd)[0] + 1
3023 else: j+=1
3023 else: j+=1
3024 else: j+=1
3024 else: j+=1
3025
3025
3026 return listMeteors
3026 return listMeteors
3027
3027
3028 def __removeMultipleDetections(self,listMeteors, rangeLimit, timeLimit):
3028 def __removeMultipleDetections(self,listMeteors, rangeLimit, timeLimit):
3029
3029
3030 arrayMeteors = numpy.asarray(listMeteors)
3030 arrayMeteors = numpy.asarray(listMeteors)
3031 listMeteors1 = []
3031 listMeteors1 = []
3032
3032
3033 while arrayMeteors.shape[0] > 0:
3033 while arrayMeteors.shape[0] > 0:
3034 FLAs = arrayMeteors[:,4]
3034 FLAs = arrayMeteors[:,4]
3035 maxFLA = FLAs.argmax()
3035 maxFLA = FLAs.argmax()
3036 listMeteors1.append(arrayMeteors[maxFLA,:])
3036 listMeteors1.append(arrayMeteors[maxFLA,:])
3037
3037
3038 MeteorInitTime = arrayMeteors[maxFLA,1]
3038 MeteorInitTime = arrayMeteors[maxFLA,1]
3039 MeteorEndTime = arrayMeteors[maxFLA,3]
3039 MeteorEndTime = arrayMeteors[maxFLA,3]
3040 MeteorHeight = arrayMeteors[maxFLA,0]
3040 MeteorHeight = arrayMeteors[maxFLA,0]
3041
3041
3042 #Check neighborhood
3042 #Check neighborhood
3043 maxHeightIndex = MeteorHeight + rangeLimit
3043 maxHeightIndex = MeteorHeight + rangeLimit
3044 minHeightIndex = MeteorHeight - rangeLimit
3044 minHeightIndex = MeteorHeight - rangeLimit
3045 minTimeIndex = MeteorInitTime - timeLimit
3045 minTimeIndex = MeteorInitTime - timeLimit
3046 maxTimeIndex = MeteorEndTime + timeLimit
3046 maxTimeIndex = MeteorEndTime + timeLimit
3047
3047
3048 #Check Heights
3048 #Check Heights
3049 indHeight = numpy.logical_and(arrayMeteors[:,0] >= minHeightIndex, arrayMeteors[:,0] <= maxHeightIndex)
3049 indHeight = numpy.logical_and(arrayMeteors[:,0] >= minHeightIndex, arrayMeteors[:,0] <= maxHeightIndex)
3050 indTime = numpy.logical_and(arrayMeteors[:,3] >= minTimeIndex, arrayMeteors[:,1] <= maxTimeIndex)
3050 indTime = numpy.logical_and(arrayMeteors[:,3] >= minTimeIndex, arrayMeteors[:,1] <= maxTimeIndex)
3051 indBoth = numpy.where(numpy.logical_and(indTime,indHeight))
3051 indBoth = numpy.where(numpy.logical_and(indTime,indHeight))
3052
3052
3053 arrayMeteors = numpy.delete(arrayMeteors, indBoth, axis = 0)
3053 arrayMeteors = numpy.delete(arrayMeteors, indBoth, axis = 0)
3054
3054
3055 return listMeteors1
3055 return listMeteors1
3056
3056
3057 def __meteorReestimation(self, listMeteors, volts, pairslist, thresh, noise, timeInterval,frequency):
3057 def __meteorReestimation(self, listMeteors, volts, pairslist, thresh, noise, timeInterval,frequency):
3058 numHeights = volts.shape[2]
3058 numHeights = volts.shape[2]
3059 nChannel = volts.shape[0]
3059 nChannel = volts.shape[0]
3060
3060
3061 thresholdPhase = thresh[0]
3061 thresholdPhase = thresh[0]
3062 thresholdNoise = thresh[1]
3062 thresholdNoise = thresh[1]
3063 thresholdDB = float(thresh[2])
3063 thresholdDB = float(thresh[2])
3064
3064
3065 thresholdDB1 = 10**(thresholdDB/10)
3065 thresholdDB1 = 10**(thresholdDB/10)
3066 pairsarray = numpy.array(pairslist)
3066 pairsarray = numpy.array(pairslist)
3067 indSides = pairsarray[:,1]
3067 indSides = pairsarray[:,1]
3068
3068
3069 pairslist1 = list(pairslist)
3069 pairslist1 = list(pairslist)
3070 pairslist1.append((0,1))
3070 pairslist1.append((0,1))
3071 pairslist1.append((3,4))
3071 pairslist1.append((3,4))
3072
3072
3073 listMeteors1 = []
3073 listMeteors1 = []
3074 listPowerSeries = []
3074 listPowerSeries = []
3075 listVoltageSeries = []
3075 listVoltageSeries = []
3076 #volts has the war data
3076 #volts has the war data
3077
3077
3078 if frequency == 30e6:
3078 if frequency == 30e6:
3079 timeLag = 45*10**-3
3079 timeLag = 45*10**-3
3080 else:
3080 else:
3081 timeLag = 15*10**-3
3081 timeLag = 15*10**-3
3082 lag = numpy.ceil(timeLag/timeInterval)
3082 lag = numpy.ceil(timeLag/timeInterval)
3083
3083
3084 for i in range(len(listMeteors)):
3084 for i in range(len(listMeteors)):
3085
3085
3086 ###################### 3.6 - 3.7 PARAMETERS REESTIMATION #########################
3086 ###################### 3.6 - 3.7 PARAMETERS REESTIMATION #########################
3087 meteorAux = numpy.zeros(16)
3087 meteorAux = numpy.zeros(16)
3088
3088
3089 #Loading meteor Data (mHeight, mStart, mPeak, mEnd)
3089 #Loading meteor Data (mHeight, mStart, mPeak, mEnd)
3090 mHeight = listMeteors[i][0]
3090 mHeight = listMeteors[i][0]
3091 mStart = listMeteors[i][1]
3091 mStart = listMeteors[i][1]
3092 mPeak = listMeteors[i][2]
3092 mPeak = listMeteors[i][2]
3093 mEnd = listMeteors[i][3]
3093 mEnd = listMeteors[i][3]
3094
3094
3095 #get the volt data between the start and end times of the meteor
3095 #get the volt data between the start and end times of the meteor
3096 meteorVolts = volts[:,mStart:mEnd+1,mHeight]
3096 meteorVolts = volts[:,mStart:mEnd+1,mHeight]
3097 meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
3097 meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
3098
3098
3099 #3.6. Phase Difference estimation
3099 #3.6. Phase Difference estimation
3100 phaseDiff, aux = self.__estimatePhaseDifference(meteorVolts, pairslist)
3100 phaseDiff, aux = self.__estimatePhaseDifference(meteorVolts, pairslist)
3101
3101
3102 #3.7. Phase difference removal & meteor start, peak and end times reestimated
3102 #3.7. Phase difference removal & meteor start, peak and end times reestimated
3103 #meteorVolts0.- all Channels, all Profiles
3103 #meteorVolts0.- all Channels, all Profiles
3104 meteorVolts0 = volts[:,:,mHeight]
3104 meteorVolts0 = volts[:,:,mHeight]
3105 meteorThresh = noise[:,mHeight]*thresholdNoise
3105 meteorThresh = noise[:,mHeight]*thresholdNoise
3106 meteorNoise = noise[:,mHeight]
3106 meteorNoise = noise[:,mHeight]
3107 meteorVolts0[indSides,:] = self.__shiftPhase(meteorVolts0[indSides,:], phaseDiff) #Phase Shifting
3107 meteorVolts0[indSides,:] = self.__shiftPhase(meteorVolts0[indSides,:], phaseDiff) #Phase Shifting
3108 powerNet0 = numpy.nansum(numpy.abs(meteorVolts0)**2, axis = 0) #Power
3108 powerNet0 = numpy.nansum(numpy.abs(meteorVolts0)**2, axis = 0) #Power
3109
3109
3110 #Times reestimation
3110 #Times reestimation
3111 mStart1 = numpy.where(powerNet0[:mPeak] < meteorThresh[:mPeak])[0]
3111 mStart1 = numpy.where(powerNet0[:mPeak] < meteorThresh[:mPeak])[0]
3112 if mStart1.size > 0:
3112 if mStart1.size > 0:
3113 mStart1 = mStart1[-1] + 1
3113 mStart1 = mStart1[-1] + 1
3114
3114
3115 else:
3115 else:
3116 mStart1 = mPeak
3116 mStart1 = mPeak
3117
3117
3118 mEnd1 = numpy.where(powerNet0[mPeak:] < meteorThresh[mPeak:])[0][0] + mPeak - 1
3118 mEnd1 = numpy.where(powerNet0[mPeak:] < meteorThresh[mPeak:])[0][0] + mPeak - 1
3119 mEndDecayTime1 = numpy.where(powerNet0[mPeak:] < meteorNoise[mPeak:])[0]
3119 mEndDecayTime1 = numpy.where(powerNet0[mPeak:] < meteorNoise[mPeak:])[0]
3120 if mEndDecayTime1.size == 0:
3120 if mEndDecayTime1.size == 0:
3121 mEndDecayTime1 = powerNet0.size
3121 mEndDecayTime1 = powerNet0.size
3122 else:
3122 else:
3123 mEndDecayTime1 = mEndDecayTime1[0] + mPeak - 1
3123 mEndDecayTime1 = mEndDecayTime1[0] + mPeak - 1
3124 # mPeak1 = meteorVolts0[mStart1:mEnd1 + 1].argmax()
3124 # mPeak1 = meteorVolts0[mStart1:mEnd1 + 1].argmax()
3125
3125
3126 #meteorVolts1.- all Channels, from start to end
3126 #meteorVolts1.- all Channels, from start to end
3127 meteorVolts1 = meteorVolts0[:,mStart1:mEnd1 + 1]
3127 meteorVolts1 = meteorVolts0[:,mStart1:mEnd1 + 1]
3128 meteorVolts2 = meteorVolts0[:,mPeak + lag:mEnd1 + 1]
3128 meteorVolts2 = meteorVolts0[:,mPeak + lag:mEnd1 + 1]
3129 if meteorVolts2.shape[1] == 0:
3129 if meteorVolts2.shape[1] == 0:
3130 meteorVolts2 = meteorVolts0[:,mPeak:mEnd1 + 1]
3130 meteorVolts2 = meteorVolts0[:,mPeak:mEnd1 + 1]
3131 meteorVolts1 = meteorVolts1.reshape(meteorVolts1.shape[0], meteorVolts1.shape[1], 1)
3131 meteorVolts1 = meteorVolts1.reshape(meteorVolts1.shape[0], meteorVolts1.shape[1], 1)
3132 meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1], 1)
3132 meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1], 1)
3133 ##################### END PARAMETERS REESTIMATION #########################
3133 ##################### END PARAMETERS REESTIMATION #########################
3134
3134
3135 ##################### 3.8 PHASE DIFFERENCE REESTIMATION ########################
3135 ##################### 3.8 PHASE DIFFERENCE REESTIMATION ########################
3136 # if mEnd1 - mStart1 > 4: #Error Number 6: echo less than 5 samples long; too short for analysis
3136 # if mEnd1 - mStart1 > 4: #Error Number 6: echo less than 5 samples long; too short for analysis
3137 if meteorVolts2.shape[1] > 0:
3137 if meteorVolts2.shape[1] > 0:
3138 #Phase Difference re-estimation
3138 #Phase Difference re-estimation
3139 phaseDiff1, phaseDiffint = self.__estimatePhaseDifference(meteorVolts2, pairslist1) #Phase Difference Estimation
3139 phaseDiff1, phaseDiffint = self.__estimatePhaseDifference(meteorVolts2, pairslist1) #Phase Difference Estimation
3140 # phaseDiff1, phaseDiffint = self.estimatePhaseDifference(meteorVolts2, pairslist)
3140 # phaseDiff1, phaseDiffint = self.estimatePhaseDifference(meteorVolts2, pairslist)
3141 meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1])
3141 meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1])
3142 phaseDiff11 = numpy.reshape(phaseDiff1, (phaseDiff1.shape[0],1))
3142 phaseDiff11 = numpy.reshape(phaseDiff1, (phaseDiff1.shape[0],1))
3143 meteorVolts2[indSides,:] = self.__shiftPhase(meteorVolts2[indSides,:], phaseDiff11[0:4]) #Phase Shifting
3143 meteorVolts2[indSides,:] = self.__shiftPhase(meteorVolts2[indSides,:], phaseDiff11[0:4]) #Phase Shifting
3144
3144
3145 #Phase Difference RMS
3145 #Phase Difference RMS
3146 phaseRMS1 = numpy.sqrt(numpy.mean(numpy.square(phaseDiff1)))
3146 phaseRMS1 = numpy.sqrt(numpy.mean(numpy.square(phaseDiff1)))
3147 powerNet1 = numpy.nansum(numpy.abs(meteorVolts1[:,:])**2,0)
3147 powerNet1 = numpy.nansum(numpy.abs(meteorVolts1[:,:])**2,0)
3148 #Data from Meteor
3148 #Data from Meteor
3149 mPeak1 = powerNet1.argmax() + mStart1
3149 mPeak1 = powerNet1.argmax() + mStart1
3150 mPeakPower1 = powerNet1.max()
3150 mPeakPower1 = powerNet1.max()
3151 noiseAux = sum(noise[mStart1:mEnd1 + 1,mHeight])
3151 noiseAux = sum(noise[mStart1:mEnd1 + 1,mHeight])
3152 mSNR1 = (sum(powerNet1)-noiseAux)/noiseAux
3152 mSNR1 = (sum(powerNet1)-noiseAux)/noiseAux
3153 Meteor1 = numpy.array([mHeight, mStart1, mPeak1, mEnd1, mPeakPower1, mSNR1, phaseRMS1])
3153 Meteor1 = numpy.array([mHeight, mStart1, mPeak1, mEnd1, mPeakPower1, mSNR1, phaseRMS1])
3154 Meteor1 = numpy.hstack((Meteor1,phaseDiffint))
3154 Meteor1 = numpy.hstack((Meteor1,phaseDiffint))
3155 PowerSeries = powerNet0[mStart1:mEndDecayTime1 + 1]
3155 PowerSeries = powerNet0[mStart1:mEndDecayTime1 + 1]
3156 #Vectorize
3156 #Vectorize
3157 meteorAux[0:7] = [mHeight, mStart1, mPeak1, mEnd1, mPeakPower1, mSNR1, phaseRMS1]
3157 meteorAux[0:7] = [mHeight, mStart1, mPeak1, mEnd1, mPeakPower1, mSNR1, phaseRMS1]
3158 meteorAux[7:11] = phaseDiffint[0:4]
3158 meteorAux[7:11] = phaseDiffint[0:4]
3159
3159
3160 #Rejection Criterions
3160 #Rejection Criterions
3161 if phaseRMS1 > thresholdPhase: #Error Number 17: Phase variation
3161 if phaseRMS1 > thresholdPhase: #Error Number 17: Phase variation
3162 meteorAux[-1] = 17
3162 meteorAux[-1] = 17
3163 elif mSNR1 < thresholdDB1: #Error Number 1: SNR < threshold dB
3163 elif mSNR1 < thresholdDB1: #Error Number 1: SNR < threshold dB
3164 meteorAux[-1] = 1
3164 meteorAux[-1] = 1
3165
3165
3166
3166
3167 else:
3167 else:
3168 meteorAux[0:4] = [mHeight, mStart, mPeak, mEnd]
3168 meteorAux[0:4] = [mHeight, mStart, mPeak, mEnd]
3169 meteorAux[-1] = 6 #Error Number 6: echo less than 5 samples long; too short for analysis
3169 meteorAux[-1] = 6 #Error Number 6: echo less than 5 samples long; too short for analysis
3170 PowerSeries = 0
3170 PowerSeries = 0
3171
3171
3172 listMeteors1.append(meteorAux)
3172 listMeteors1.append(meteorAux)
3173 listPowerSeries.append(PowerSeries)
3173 listPowerSeries.append(PowerSeries)
3174 listVoltageSeries.append(meteorVolts1)
3174 listVoltageSeries.append(meteorVolts1)
3175
3175
3176 return listMeteors1, listPowerSeries, listVoltageSeries
3176 return listMeteors1, listPowerSeries, listVoltageSeries
3177
3177
3178 def __estimateDecayTime(self, listMeteors, listPower, timeInterval, frequency):
3178 def __estimateDecayTime(self, listMeteors, listPower, timeInterval, frequency):
3179
3179
3180 threshError = 10
3180 threshError = 10
3181 #Depending if it is 30 or 50 MHz
3181 #Depending if it is 30 or 50 MHz
3182 if frequency == 30e6:
3182 if frequency == 30e6:
3183 timeLag = 45*10**-3
3183 timeLag = 45*10**-3
3184 else:
3184 else:
3185 timeLag = 15*10**-3
3185 timeLag = 15*10**-3
3186 lag = numpy.ceil(timeLag/timeInterval)
3186 lag = numpy.ceil(timeLag/timeInterval)
3187
3187
3188 listMeteors1 = []
3188 listMeteors1 = []
3189
3189
3190 for i in range(len(listMeteors)):
3190 for i in range(len(listMeteors)):
3191 meteorPower = listPower[i]
3191 meteorPower = listPower[i]
3192 meteorAux = listMeteors[i]
3192 meteorAux = listMeteors[i]
3193
3193
3194 if meteorAux[-1] == 0:
3194 if meteorAux[-1] == 0:
3195
3195
3196 try:
3196 try:
3197 indmax = meteorPower.argmax()
3197 indmax = meteorPower.argmax()
3198 indlag = indmax + lag
3198 indlag = indmax + lag
3199
3199
3200 y = meteorPower[indlag:]
3200 y = meteorPower[indlag:]
3201 x = numpy.arange(0, y.size)*timeLag
3201 x = numpy.arange(0, y.size)*timeLag
3202
3202
3203 #first guess
3203 #first guess
3204 a = y[0]
3204 a = y[0]
3205 tau = timeLag
3205 tau = timeLag
3206 #exponential fit
3206 #exponential fit
3207 popt, pcov = optimize.curve_fit(self.__exponential_function, x, y, p0 = [a, tau])
3207 popt, pcov = optimize.curve_fit(self.__exponential_function, x, y, p0 = [a, tau])
3208 y1 = self.__exponential_function(x, *popt)
3208 y1 = self.__exponential_function(x, *popt)
3209 #error estimation
3209 #error estimation
3210 error = sum((y - y1)**2)/(numpy.var(y)*(y.size - popt.size))
3210 error = sum((y - y1)**2)/(numpy.var(y)*(y.size - popt.size))
3211
3211
3212 decayTime = popt[1]
3212 decayTime = popt[1]
3213 riseTime = indmax*timeInterval
3213 riseTime = indmax*timeInterval
3214 meteorAux[11:13] = [decayTime, error]
3214 meteorAux[11:13] = [decayTime, error]
3215
3215
3216 #Table items 7, 8 and 11
3216 #Table items 7, 8 and 11
3217 if (riseTime > 0.3): #Number 7: Echo rise exceeds 0.3s
3217 if (riseTime > 0.3): #Number 7: Echo rise exceeds 0.3s
3218 meteorAux[-1] = 7
3218 meteorAux[-1] = 7
3219 elif (decayTime < 2*riseTime) : #Number 8: Echo decay time less than than twice rise time
3219 elif (decayTime < 2*riseTime) : #Number 8: Echo decay time less than than twice rise time
3220 meteorAux[-1] = 8
3220 meteorAux[-1] = 8
3221 if (error > threshError): #Number 11: Poor fit to amplitude for estimation of decay time
3221 if (error > threshError): #Number 11: Poor fit to amplitude for estimation of decay time
3222 meteorAux[-1] = 11
3222 meteorAux[-1] = 11
3223
3223
3224
3224
3225 except:
3225 except:
3226 meteorAux[-1] = 11
3226 meteorAux[-1] = 11
3227
3227
3228
3228
3229 listMeteors1.append(meteorAux)
3229 listMeteors1.append(meteorAux)
3230
3230
3231 return listMeteors1
3231 return listMeteors1
3232
3232
3233 #Exponential Function
3233 #Exponential Function
3234
3234
3235 def __exponential_function(self, x, a, tau):
3235 def __exponential_function(self, x, a, tau):
3236 y = a*numpy.exp(-x/tau)
3236 y = a*numpy.exp(-x/tau)
3237 return y
3237 return y
3238
3238
3239 def __getRadialVelocity(self, listMeteors, listVolts, radialStdThresh, pairslist, timeInterval):
3239 def __getRadialVelocity(self, listMeteors, listVolts, radialStdThresh, pairslist, timeInterval):
3240
3240
3241 pairslist1 = list(pairslist)
3241 pairslist1 = list(pairslist)
3242 pairslist1.append((0,1))
3242 pairslist1.append((0,1))
3243 pairslist1.append((3,4))
3243 pairslist1.append((3,4))
3244 numPairs = len(pairslist1)
3244 numPairs = len(pairslist1)
3245 #Time Lag
3245 #Time Lag
3246 timeLag = 45*10**-3
3246 timeLag = 45*10**-3
3247 c = 3e8
3247 c = 3e8
3248 lag = numpy.ceil(timeLag/timeInterval)
3248 lag = numpy.ceil(timeLag/timeInterval)
3249 freq = 30e6
3249 freq = 30e6
3250
3250
3251 listMeteors1 = []
3251 listMeteors1 = []
3252
3252
3253 for i in range(len(listMeteors)):
3253 for i in range(len(listMeteors)):
3254 meteorAux = listMeteors[i]
3254 meteorAux = listMeteors[i]
3255 if meteorAux[-1] == 0:
3255 if meteorAux[-1] == 0:
3256 mStart = listMeteors[i][1]
3256 mStart = listMeteors[i][1]
3257 mPeak = listMeteors[i][2]
3257 mPeak = listMeteors[i][2]
3258 mLag = mPeak - mStart + lag
3258 mLag = mPeak - mStart + lag
3259
3259
3260 #get the volt data between the start and end times of the meteor
3260 #get the volt data between the start and end times of the meteor
3261 meteorVolts = listVolts[i]
3261 meteorVolts = listVolts[i]
3262 meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
3262 meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
3263
3263
3264 #Get CCF
3264 #Get CCF
3265 allCCFs = self.__calculateCCF(meteorVolts, pairslist1, [-2,-1,0,1,2])
3265 allCCFs = self.__calculateCCF(meteorVolts, pairslist1, [-2,-1,0,1,2])
3266
3266
3267 #Method 2
3267 #Method 2
3268 slopes = numpy.zeros(numPairs)
3268 slopes = numpy.zeros(numPairs)
3269 time = numpy.array([-2,-1,1,2])*timeInterval
3269 time = numpy.array([-2,-1,1,2])*timeInterval
3270 angAllCCF = numpy.angle(allCCFs[:,[0,1,3,4],0])
3270 angAllCCF = numpy.angle(allCCFs[:,[0,1,3,4],0])
3271
3271
3272 #Correct phases
3272 #Correct phases
3273 derPhaseCCF = angAllCCF[:,1:] - angAllCCF[:,0:-1]
3273 derPhaseCCF = angAllCCF[:,1:] - angAllCCF[:,0:-1]
3274 indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
3274 indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
3275
3275
3276 if indDer[0].shape[0] > 0:
3276 if indDer[0].shape[0] > 0:
3277 for i in range(indDer[0].shape[0]):
3277 for i in range(indDer[0].shape[0]):
3278 signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i]])
3278 signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i]])
3279 angAllCCF[indDer[0][i],indDer[1][i]+1:] += signo*2*numpy.pi
3279 angAllCCF[indDer[0][i],indDer[1][i]+1:] += signo*2*numpy.pi
3280
3280
3281 # fit = scipy.stats.linregress(numpy.array([-2,-1,1,2])*timeInterval, numpy.array([phaseLagN2s[i],phaseLagN1s[i],phaseLag1s[i],phaseLag2s[i]]))
3281 # fit = scipy.stats.linregress(numpy.array([-2,-1,1,2])*timeInterval, numpy.array([phaseLagN2s[i],phaseLagN1s[i],phaseLag1s[i],phaseLag2s[i]]))
3282 for j in range(numPairs):
3282 for j in range(numPairs):
3283 fit = stats.linregress(time, angAllCCF[j,:])
3283 fit = stats.linregress(time, angAllCCF[j,:])
3284 slopes[j] = fit[0]
3284 slopes[j] = fit[0]
3285
3285
3286 #Remove Outlier
3286 #Remove Outlier
3287 # indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
3287 # indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
3288 # slopes = numpy.delete(slopes,indOut)
3288 # slopes = numpy.delete(slopes,indOut)
3289 # indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
3289 # indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
3290 # slopes = numpy.delete(slopes,indOut)
3290 # slopes = numpy.delete(slopes,indOut)
3291
3291
3292 radialVelocity = -numpy.mean(slopes)*(0.25/numpy.pi)*(c/freq)
3292 radialVelocity = -numpy.mean(slopes)*(0.25/numpy.pi)*(c/freq)
3293 radialError = numpy.std(slopes)*(0.25/numpy.pi)*(c/freq)
3293 radialError = numpy.std(slopes)*(0.25/numpy.pi)*(c/freq)
3294 meteorAux[-2] = radialError
3294 meteorAux[-2] = radialError
3295 meteorAux[-3] = radialVelocity
3295 meteorAux[-3] = radialVelocity
3296
3296
3297 #Setting Error
3297 #Setting Error
3298 #Number 15: Radial Drift velocity or projected horizontal velocity exceeds 200 m/s
3298 #Number 15: Radial Drift velocity or projected horizontal velocity exceeds 200 m/s
3299 if numpy.abs(radialVelocity) > 200:
3299 if numpy.abs(radialVelocity) > 200:
3300 meteorAux[-1] = 15
3300 meteorAux[-1] = 15
3301 #Number 12: Poor fit to CCF variation for estimation of radial drift velocity
3301 #Number 12: Poor fit to CCF variation for estimation of radial drift velocity
3302 elif radialError > radialStdThresh:
3302 elif radialError > radialStdThresh:
3303 meteorAux[-1] = 12
3303 meteorAux[-1] = 12
3304
3304
3305 listMeteors1.append(meteorAux)
3305 listMeteors1.append(meteorAux)
3306 return listMeteors1
3306 return listMeteors1
3307
3307
3308 def __setNewArrays(self, listMeteors, date, heiRang):
3308 def __setNewArrays(self, listMeteors, date, heiRang):
3309
3309
3310 #New arrays
3310 #New arrays
3311 arrayMeteors = numpy.array(listMeteors)
3311 arrayMeteors = numpy.array(listMeteors)
3312 arrayParameters = numpy.zeros((len(listMeteors), 13))
3312 arrayParameters = numpy.zeros((len(listMeteors), 13))
3313
3313
3314 #Date inclusion
3314 #Date inclusion
3315 # date = re.findall(r'\((.*?)\)', date)
3315 # date = re.findall(r'\((.*?)\)', date)
3316 # date = date[0].split(',')
3316 # date = date[0].split(',')
3317 # date = map(int, date)
3317 # date = map(int, date)
3318 #
3318 #
3319 # if len(date)<6:
3319 # if len(date)<6:
3320 # date.append(0)
3320 # date.append(0)
3321 #
3321 #
3322 # date = [date[0]*10000 + date[1]*100 + date[2], date[3]*10000 + date[4]*100 + date[5]]
3322 # date = [date[0]*10000 + date[1]*100 + date[2], date[3]*10000 + date[4]*100 + date[5]]
3323 # arrayDate = numpy.tile(date, (len(listMeteors), 1))
3323 # arrayDate = numpy.tile(date, (len(listMeteors), 1))
3324 arrayDate = numpy.tile(date, (len(listMeteors)))
3324 arrayDate = numpy.tile(date, (len(listMeteors)))
3325
3325
3326 #Meteor array
3326 #Meteor array
3327 # arrayMeteors[:,0] = heiRang[arrayMeteors[:,0].astype(int)]
3327 # arrayMeteors[:,0] = heiRang[arrayMeteors[:,0].astype(int)]
3328 # arrayMeteors = numpy.hstack((arrayDate, arrayMeteors))
3328 # arrayMeteors = numpy.hstack((arrayDate, arrayMeteors))
3329
3329
3330 #Parameters Array
3330 #Parameters Array
3331 arrayParameters[:,0] = arrayDate #Date
3331 arrayParameters[:,0] = arrayDate #Date
3332 arrayParameters[:,1] = heiRang[arrayMeteors[:,0].astype(int)] #Range
3332 arrayParameters[:,1] = heiRang[arrayMeteors[:,0].astype(int)] #Range
3333 arrayParameters[:,6:8] = arrayMeteors[:,-3:-1] #Radial velocity and its error
3333 arrayParameters[:,6:8] = arrayMeteors[:,-3:-1] #Radial velocity and its error
3334 arrayParameters[:,8:12] = arrayMeteors[:,7:11] #Phases
3334 arrayParameters[:,8:12] = arrayMeteors[:,7:11] #Phases
3335 arrayParameters[:,-1] = arrayMeteors[:,-1] #Error
3335 arrayParameters[:,-1] = arrayMeteors[:,-1] #Error
3336
3336
3337
3337
3338 return arrayParameters
3338 return arrayParameters
3339
3339
3340 class CorrectSMPhases(Operation):
3340 class CorrectSMPhases(Operation):
3341
3341
3342 def run(self, dataOut, phaseOffsets, hmin = 50, hmax = 150, azimuth = 45, channelPositions = None):
3342 def run(self, dataOut, phaseOffsets, hmin = 50, hmax = 150, azimuth = 45, channelPositions = None):
3343
3343
3344 arrayParameters = dataOut.data_param
3344 arrayParameters = dataOut.data_param
3345 pairsList = []
3345 pairsList = []
3346 pairx = (0,1)
3346 pairx = (0,1)
3347 pairy = (2,3)
3347 pairy = (2,3)
3348 pairsList.append(pairx)
3348 pairsList.append(pairx)
3349 pairsList.append(pairy)
3349 pairsList.append(pairy)
3350 jph = numpy.zeros(4)
3350 jph = numpy.zeros(4)
3351
3351
3352 phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
3352 phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
3353 # arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
3353 # arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
3354 arrayParameters[:,8:12] = numpy.angle(numpy.exp(1j*(arrayParameters[:,8:12] + phaseOffsets)))
3354 arrayParameters[:,8:12] = numpy.angle(numpy.exp(1j*(arrayParameters[:,8:12] + phaseOffsets)))
3355
3355
3356 meteorOps = SMOperations()
3356 meteorOps = SMOperations()
3357 if channelPositions is None:
3357 if channelPositions is None:
3358 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
3358 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
3359 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
3359 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
3360
3360
3361 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
3361 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
3362 h = (hmin,hmax)
3362 h = (hmin,hmax)
3363
3363
3364 arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
3364 arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
3365
3365
3366 dataOut.data_param = arrayParameters
3366 dataOut.data_param = arrayParameters
3367 return
3367 return
3368
3368
3369 class SMPhaseCalibration(Operation):
3369 class SMPhaseCalibration(Operation):
3370
3370
3371 __buffer = None
3371 __buffer = None
3372
3372
3373 __initime = None
3373 __initime = None
3374
3374
3375 __dataReady = False
3375 __dataReady = False
3376
3376
3377 __isConfig = False
3377 __isConfig = False
3378
3378
3379 def __checkTime(self, currentTime, initTime, paramInterval, outputInterval):
3379 def __checkTime(self, currentTime, initTime, paramInterval, outputInterval):
3380
3380
3381 dataTime = currentTime + paramInterval
3381 dataTime = currentTime + paramInterval
3382 deltaTime = dataTime - initTime
3382 deltaTime = dataTime - initTime
3383
3383
3384 if deltaTime >= outputInterval or deltaTime < 0:
3384 if deltaTime >= outputInterval or deltaTime < 0:
3385 return True
3385 return True
3386
3386
3387 return False
3387 return False
3388
3388
3389 def __getGammas(self, pairs, d, phases):
3389 def __getGammas(self, pairs, d, phases):
3390 gammas = numpy.zeros(2)
3390 gammas = numpy.zeros(2)
3391
3391
3392 for i in range(len(pairs)):
3392 for i in range(len(pairs)):
3393
3393
3394 pairi = pairs[i]
3394 pairi = pairs[i]
3395
3395
3396 phip3 = phases[:,pairi[0]]
3396 phip3 = phases[:,pairi[0]]
3397 d3 = d[pairi[0]]
3397 d3 = d[pairi[0]]
3398 phip2 = phases[:,pairi[1]]
3398 phip2 = phases[:,pairi[1]]
3399 d2 = d[pairi[1]]
3399 d2 = d[pairi[1]]
3400 #Calculating gamma
3400 #Calculating gamma
3401 # jdcos = alp1/(k*d1)
3401 # jdcos = alp1/(k*d1)
3402 # jgamma = numpy.angle(numpy.exp(1j*(d0*alp1/d1 - alp0)))
3402 # jgamma = numpy.angle(numpy.exp(1j*(d0*alp1/d1 - alp0)))
3403 jgamma = -phip2*d3/d2 - phip3
3403 jgamma = -phip2*d3/d2 - phip3
3404 jgamma = numpy.angle(numpy.exp(1j*jgamma))
3404 jgamma = numpy.angle(numpy.exp(1j*jgamma))
3405 # jgamma[jgamma>numpy.pi] -= 2*numpy.pi
3405 # jgamma[jgamma>numpy.pi] -= 2*numpy.pi
3406 # jgamma[jgamma<-numpy.pi] += 2*numpy.pi
3406 # jgamma[jgamma<-numpy.pi] += 2*numpy.pi
3407
3407
3408 #Revised distribution
3408 #Revised distribution
3409 jgammaArray = numpy.hstack((jgamma,jgamma+0.5*numpy.pi,jgamma-0.5*numpy.pi))
3409 jgammaArray = numpy.hstack((jgamma,jgamma+0.5*numpy.pi,jgamma-0.5*numpy.pi))
3410
3410
3411 #Histogram
3411 #Histogram
3412 nBins = 64
3412 nBins = 64
3413 rmin = -0.5*numpy.pi
3413 rmin = -0.5*numpy.pi
3414 rmax = 0.5*numpy.pi
3414 rmax = 0.5*numpy.pi
3415 phaseHisto = numpy.histogram(jgammaArray, bins=nBins, range=(rmin,rmax))
3415 phaseHisto = numpy.histogram(jgammaArray, bins=nBins, range=(rmin,rmax))
3416
3416
3417 meteorsY = phaseHisto[0]
3417 meteorsY = phaseHisto[0]
3418 phasesX = phaseHisto[1][:-1]
3418 phasesX = phaseHisto[1][:-1]
3419 width = phasesX[1] - phasesX[0]
3419 width = phasesX[1] - phasesX[0]
3420 phasesX += width/2
3420 phasesX += width/2
3421
3421
3422 #Gaussian aproximation
3422 #Gaussian aproximation
3423 bpeak = meteorsY.argmax()
3423 bpeak = meteorsY.argmax()
3424 peak = meteorsY.max()
3424 peak = meteorsY.max()
3425 jmin = bpeak - 5
3425 jmin = bpeak - 5
3426 jmax = bpeak + 5 + 1
3426 jmax = bpeak + 5 + 1
3427
3427
3428 if jmin<0:
3428 if jmin<0:
3429 jmin = 0
3429 jmin = 0
3430 jmax = 6
3430 jmax = 6
3431 elif jmax > meteorsY.size:
3431 elif jmax > meteorsY.size:
3432 jmin = meteorsY.size - 6
3432 jmin = meteorsY.size - 6
3433 jmax = meteorsY.size
3433 jmax = meteorsY.size
3434
3434
3435 x0 = numpy.array([peak,bpeak,50])
3435 x0 = numpy.array([peak,bpeak,50])
3436 coeff = optimize.leastsq(self.__residualFunction, x0, args=(meteorsY[jmin:jmax], phasesX[jmin:jmax]))
3436 coeff = optimize.leastsq(self.__residualFunction, x0, args=(meteorsY[jmin:jmax], phasesX[jmin:jmax]))
3437
3437
3438 #Gammas
3438 #Gammas
3439 gammas[i] = coeff[0][1]
3439 gammas[i] = coeff[0][1]
3440
3440
3441 return gammas
3441 return gammas
3442
3442
3443 def __residualFunction(self, coeffs, y, t):
3443 def __residualFunction(self, coeffs, y, t):
3444
3444
3445 return y - self.__gauss_function(t, coeffs)
3445 return y - self.__gauss_function(t, coeffs)
3446
3446
3447 def __gauss_function(self, t, coeffs):
3447 def __gauss_function(self, t, coeffs):
3448
3448
3449 return coeffs[0]*numpy.exp(-0.5*((t - coeffs[1]) / coeffs[2])**2)
3449 return coeffs[0]*numpy.exp(-0.5*((t - coeffs[1]) / coeffs[2])**2)
3450
3450
3451 def __getPhases(self, azimuth, h, pairsList, d, gammas, meteorsArray):
3451 def __getPhases(self, azimuth, h, pairsList, d, gammas, meteorsArray):
3452 meteorOps = SMOperations()
3452 meteorOps = SMOperations()
3453 nchan = 4
3453 nchan = 4
3454 pairx = pairsList[0] #x es 0
3454 pairx = pairsList[0] #x es 0
3455 pairy = pairsList[1] #y es 1
3455 pairy = pairsList[1] #y es 1
3456 center_xangle = 0
3456 center_xangle = 0
3457 center_yangle = 0
3457 center_yangle = 0
3458 range_angle = numpy.array([10*numpy.pi,numpy.pi,numpy.pi/2,numpy.pi/4])
3458 range_angle = numpy.array([10*numpy.pi,numpy.pi,numpy.pi/2,numpy.pi/4])
3459 ntimes = len(range_angle)
3459 ntimes = len(range_angle)
3460
3460
3461 nstepsx = 20
3461 nstepsx = 20
3462 nstepsy = 20
3462 nstepsy = 20
3463
3463
3464 for iz in range(ntimes):
3464 for iz in range(ntimes):
3465 min_xangle = -range_angle[iz]/2 + center_xangle
3465 min_xangle = -range_angle[iz]/2 + center_xangle
3466 max_xangle = range_angle[iz]/2 + center_xangle
3466 max_xangle = range_angle[iz]/2 + center_xangle
3467 min_yangle = -range_angle[iz]/2 + center_yangle
3467 min_yangle = -range_angle[iz]/2 + center_yangle
3468 max_yangle = range_angle[iz]/2 + center_yangle
3468 max_yangle = range_angle[iz]/2 + center_yangle
3469
3469
3470 inc_x = (max_xangle-min_xangle)/nstepsx
3470 inc_x = (max_xangle-min_xangle)/nstepsx
3471 inc_y = (max_yangle-min_yangle)/nstepsy
3471 inc_y = (max_yangle-min_yangle)/nstepsy
3472
3472
3473 alpha_y = numpy.arange(nstepsy)*inc_y + min_yangle
3473 alpha_y = numpy.arange(nstepsy)*inc_y + min_yangle
3474 alpha_x = numpy.arange(nstepsx)*inc_x + min_xangle
3474 alpha_x = numpy.arange(nstepsx)*inc_x + min_xangle
3475 penalty = numpy.zeros((nstepsx,nstepsy))
3475 penalty = numpy.zeros((nstepsx,nstepsy))
3476 jph_array = numpy.zeros((nchan,nstepsx,nstepsy))
3476 jph_array = numpy.zeros((nchan,nstepsx,nstepsy))
3477 jph = numpy.zeros(nchan)
3477 jph = numpy.zeros(nchan)
3478
3478
3479 # Iterations looking for the offset
3479 # Iterations looking for the offset
3480 for iy in range(int(nstepsy)):
3480 for iy in range(int(nstepsy)):
3481 for ix in range(int(nstepsx)):
3481 for ix in range(int(nstepsx)):
3482 d3 = d[pairsList[1][0]]
3482 d3 = d[pairsList[1][0]]
3483 d2 = d[pairsList[1][1]]
3483 d2 = d[pairsList[1][1]]
3484 d5 = d[pairsList[0][0]]
3484 d5 = d[pairsList[0][0]]
3485 d4 = d[pairsList[0][1]]
3485 d4 = d[pairsList[0][1]]
3486
3486
3487 alp2 = alpha_y[iy] #gamma 1
3487 alp2 = alpha_y[iy] #gamma 1
3488 alp4 = alpha_x[ix] #gamma 0
3488 alp4 = alpha_x[ix] #gamma 0
3489
3489
3490 alp3 = -alp2*d3/d2 - gammas[1]
3490 alp3 = -alp2*d3/d2 - gammas[1]
3491 alp5 = -alp4*d5/d4 - gammas[0]
3491 alp5 = -alp4*d5/d4 - gammas[0]
3492 # jph[pairy[1]] = alpha_y[iy]
3492 # jph[pairy[1]] = alpha_y[iy]
3493 # jph[pairy[0]] = -gammas[1] - alpha_y[iy]*d[pairy[1]]/d[pairy[0]]
3493 # jph[pairy[0]] = -gammas[1] - alpha_y[iy]*d[pairy[1]]/d[pairy[0]]
3494
3494
3495 # jph[pairx[1]] = alpha_x[ix]
3495 # jph[pairx[1]] = alpha_x[ix]
3496 # jph[pairx[0]] = -gammas[0] - alpha_x[ix]*d[pairx[1]]/d[pairx[0]]
3496 # jph[pairx[0]] = -gammas[0] - alpha_x[ix]*d[pairx[1]]/d[pairx[0]]
3497 jph[pairsList[0][1]] = alp4
3497 jph[pairsList[0][1]] = alp4
3498 jph[pairsList[0][0]] = alp5
3498 jph[pairsList[0][0]] = alp5
3499 jph[pairsList[1][0]] = alp3
3499 jph[pairsList[1][0]] = alp3
3500 jph[pairsList[1][1]] = alp2
3500 jph[pairsList[1][1]] = alp2
3501 jph_array[:,ix,iy] = jph
3501 jph_array[:,ix,iy] = jph
3502 # d = [2.0,2.5,2.5,2.0]
3502 # d = [2.0,2.5,2.5,2.0]
3503 #falta chequear si va a leer bien los meteoros
3503 #falta chequear si va a leer bien los meteoros
3504 meteorsArray1 = meteorOps.getMeteorParams(meteorsArray, azimuth, h, pairsList, d, jph)
3504 meteorsArray1 = meteorOps.getMeteorParams(meteorsArray, azimuth, h, pairsList, d, jph)
3505 error = meteorsArray1[:,-1]
3505 error = meteorsArray1[:,-1]
3506 ind1 = numpy.where(error==0)[0]
3506 ind1 = numpy.where(error==0)[0]
3507 penalty[ix,iy] = ind1.size
3507 penalty[ix,iy] = ind1.size
3508
3508
3509 i,j = numpy.unravel_index(penalty.argmax(), penalty.shape)
3509 i,j = numpy.unravel_index(penalty.argmax(), penalty.shape)
3510 phOffset = jph_array[:,i,j]
3510 phOffset = jph_array[:,i,j]
3511
3511
3512 center_xangle = phOffset[pairx[1]]
3512 center_xangle = phOffset[pairx[1]]
3513 center_yangle = phOffset[pairy[1]]
3513 center_yangle = phOffset[pairy[1]]
3514
3514
3515 phOffset = numpy.angle(numpy.exp(1j*jph_array[:,i,j]))
3515 phOffset = numpy.angle(numpy.exp(1j*jph_array[:,i,j]))
3516 phOffset = phOffset*180/numpy.pi
3516 phOffset = phOffset*180/numpy.pi
3517 return phOffset
3517 return phOffset
3518
3518
3519
3519
3520 def run(self, dataOut, hmin, hmax, channelPositions=None, nHours = 1):
3520 def run(self, dataOut, hmin, hmax, channelPositions=None, nHours = 1):
3521
3521
3522 dataOut.flagNoData = True
3522 dataOut.flagNoData = True
3523 self.__dataReady = False
3523 self.__dataReady = False
3524 dataOut.outputInterval = nHours*3600
3524 dataOut.outputInterval = nHours*3600
3525
3525
3526 if self.__isConfig == False:
3526 if self.__isConfig == False:
3527 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
3527 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
3528 #Get Initial LTC time
3528 #Get Initial LTC time
3529 self.__initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
3529 self.__initime = datetime.datetime.utcfromtimestamp(dataOut.utctime)
3530 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
3530 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
3531
3531
3532 self.__isConfig = True
3532 self.__isConfig = True
3533
3533
3534 if self.__buffer is None:
3534 if self.__buffer is None:
3535 self.__buffer = dataOut.data_param.copy()
3535 self.__buffer = dataOut.data_param.copy()
3536
3536
3537 else:
3537 else:
3538 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
3538 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
3539
3539
3540 self.__dataReady = self.__checkTime(dataOut.utctime, self.__initime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
3540 self.__dataReady = self.__checkTime(dataOut.utctime, self.__initime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
3541
3541
3542 if self.__dataReady:
3542 if self.__dataReady:
3543 dataOut.utctimeInit = self.__initime
3543 dataOut.utctimeInit = self.__initime
3544 self.__initime += dataOut.outputInterval #to erase time offset
3544 self.__initime += dataOut.outputInterval #to erase time offset
3545
3545
3546 freq = dataOut.frequency
3546 freq = dataOut.frequency
3547 c = dataOut.C #m/s
3547 c = dataOut.C #m/s
3548 lamb = c/freq
3548 lamb = c/freq
3549 k = 2*numpy.pi/lamb
3549 k = 2*numpy.pi/lamb
3550 azimuth = 0
3550 azimuth = 0
3551 h = (hmin, hmax)
3551 h = (hmin, hmax)
3552 # pairs = ((0,1),(2,3)) #Estrella
3552 # pairs = ((0,1),(2,3)) #Estrella
3553 # pairs = ((1,0),(2,3)) #T
3553 # pairs = ((1,0),(2,3)) #T
3554
3554
3555 if channelPositions is None:
3555 if channelPositions is None:
3556 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
3556 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
3557 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
3557 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
3558 meteorOps = SMOperations()
3558 meteorOps = SMOperations()
3559 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
3559 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
3560
3560
3561 #Checking correct order of pairs
3561 #Checking correct order of pairs
3562 pairs = []
3562 pairs = []
3563 if distances[1] > distances[0]:
3563 if distances[1] > distances[0]:
3564 pairs.append((1,0))
3564 pairs.append((1,0))
3565 else:
3565 else:
3566 pairs.append((0,1))
3566 pairs.append((0,1))
3567
3567
3568 if distances[3] > distances[2]:
3568 if distances[3] > distances[2]:
3569 pairs.append((3,2))
3569 pairs.append((3,2))
3570 else:
3570 else:
3571 pairs.append((2,3))
3571 pairs.append((2,3))
3572 # distances1 = [-distances[0]*lamb, distances[1]*lamb, -distances[2]*lamb, distances[3]*lamb]
3572 # distances1 = [-distances[0]*lamb, distances[1]*lamb, -distances[2]*lamb, distances[3]*lamb]
3573
3573
3574 meteorsArray = self.__buffer
3574 meteorsArray = self.__buffer
3575 error = meteorsArray[:,-1]
3575 error = meteorsArray[:,-1]
3576 boolError = (error==0)|(error==3)|(error==4)|(error==13)|(error==14)
3576 boolError = (error==0)|(error==3)|(error==4)|(error==13)|(error==14)
3577 ind1 = numpy.where(boolError)[0]
3577 ind1 = numpy.where(boolError)[0]
3578 meteorsArray = meteorsArray[ind1,:]
3578 meteorsArray = meteorsArray[ind1,:]
3579 meteorsArray[:,-1] = 0
3579 meteorsArray[:,-1] = 0
3580 phases = meteorsArray[:,8:12]
3580 phases = meteorsArray[:,8:12]
3581
3581
3582 #Calculate Gammas
3582 #Calculate Gammas
3583 gammas = self.__getGammas(pairs, distances, phases)
3583 gammas = self.__getGammas(pairs, distances, phases)
3584 # gammas = numpy.array([-21.70409463,45.76935864])*numpy.pi/180
3584 # gammas = numpy.array([-21.70409463,45.76935864])*numpy.pi/180
3585 #Calculate Phases
3585 #Calculate Phases
3586 phasesOff = self.__getPhases(azimuth, h, pairs, distances, gammas, meteorsArray)
3586 phasesOff = self.__getPhases(azimuth, h, pairs, distances, gammas, meteorsArray)
3587 phasesOff = phasesOff.reshape((1,phasesOff.size))
3587 phasesOff = phasesOff.reshape((1,phasesOff.size))
3588 dataOut.data_output = -phasesOff
3588 dataOut.data_output = -phasesOff
3589 dataOut.flagNoData = False
3589 dataOut.flagNoData = False
3590 self.__buffer = None
3590 self.__buffer = None
3591
3591
3592
3592
3593 return
3593 return
3594
3594
3595 class SMOperations():
3595 class SMOperations():
3596
3596
3597 def __init__(self):
3597 def __init__(self):
3598
3598
3599 return
3599 return
3600
3600
3601 def getMeteorParams(self, arrayParameters0, azimuth, h, pairsList, distances, jph):
3601 def getMeteorParams(self, arrayParameters0, azimuth, h, pairsList, distances, jph):
3602
3602
3603 arrayParameters = arrayParameters0.copy()
3603 arrayParameters = arrayParameters0.copy()
3604 hmin = h[0]
3604 hmin = h[0]
3605 hmax = h[1]
3605 hmax = h[1]
3606
3606
3607 #Calculate AOA (Error N 3, 4)
3607 #Calculate AOA (Error N 3, 4)
3608 #JONES ET AL. 1998
3608 #JONES ET AL. 1998
3609 AOAthresh = numpy.pi/8
3609 AOAthresh = numpy.pi/8
3610 error = arrayParameters[:,-1]
3610 error = arrayParameters[:,-1]
3611 phases = -arrayParameters[:,8:12] + jph
3611 phases = -arrayParameters[:,8:12] + jph
3612 # phases = numpy.unwrap(phases)
3612 # phases = numpy.unwrap(phases)
3613 arrayParameters[:,3:6], arrayParameters[:,-1] = self.__getAOA(phases, pairsList, distances, error, AOAthresh, azimuth)
3613 arrayParameters[:,3:6], arrayParameters[:,-1] = self.__getAOA(phases, pairsList, distances, error, AOAthresh, azimuth)
3614
3614
3615 #Calculate Heights (Error N 13 and 14)
3615 #Calculate Heights (Error N 13 and 14)
3616 error = arrayParameters[:,-1]
3616 error = arrayParameters[:,-1]
3617 Ranges = arrayParameters[:,1]
3617 Ranges = arrayParameters[:,1]
3618 zenith = arrayParameters[:,4]
3618 zenith = arrayParameters[:,4]
3619 arrayParameters[:,2], arrayParameters[:,-1] = self.__getHeights(Ranges, zenith, error, hmin, hmax)
3619 arrayParameters[:,2], arrayParameters[:,-1] = self.__getHeights(Ranges, zenith, error, hmin, hmax)
3620
3620
3621 #----------------------- Get Final data ------------------------------------
3621 #----------------------- Get Final data ------------------------------------
3622 # error = arrayParameters[:,-1]
3622 # error = arrayParameters[:,-1]
3623 # ind1 = numpy.where(error==0)[0]
3623 # ind1 = numpy.where(error==0)[0]
3624 # arrayParameters = arrayParameters[ind1,:]
3624 # arrayParameters = arrayParameters[ind1,:]
3625
3625
3626 return arrayParameters
3626 return arrayParameters
3627
3627
3628 def __getAOA(self, phases, pairsList, directions, error, AOAthresh, azimuth):
3628 def __getAOA(self, phases, pairsList, directions, error, AOAthresh, azimuth):
3629
3629
3630 arrayAOA = numpy.zeros((phases.shape[0],3))
3630 arrayAOA = numpy.zeros((phases.shape[0],3))
3631 cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList,directions)
3631 cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList,directions)
3632
3632
3633 arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
3633 arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
3634 cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
3634 cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
3635 arrayAOA[:,2] = cosDirError
3635 arrayAOA[:,2] = cosDirError
3636
3636
3637 azimuthAngle = arrayAOA[:,0]
3637 azimuthAngle = arrayAOA[:,0]
3638 zenithAngle = arrayAOA[:,1]
3638 zenithAngle = arrayAOA[:,1]
3639
3639
3640 #Setting Error
3640 #Setting Error
3641 indError = numpy.where(numpy.logical_or(error == 3, error == 4))[0]
3641 indError = numpy.where(numpy.logical_or(error == 3, error == 4))[0]
3642 error[indError] = 0
3642 error[indError] = 0
3643 #Number 3: AOA not fesible
3643 #Number 3: AOA not fesible
3644 indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
3644 indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
3645 error[indInvalid] = 3
3645 error[indInvalid] = 3
3646 #Number 4: Large difference in AOAs obtained from different antenna baselines
3646 #Number 4: Large difference in AOAs obtained from different antenna baselines
3647 indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
3647 indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
3648 error[indInvalid] = 4
3648 error[indInvalid] = 4
3649 return arrayAOA, error
3649 return arrayAOA, error
3650
3650
3651 def __getDirectionCosines(self, arrayPhase, pairsList, distances):
3651 def __getDirectionCosines(self, arrayPhase, pairsList, distances):
3652
3652
3653 #Initializing some variables
3653 #Initializing some variables
3654 ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
3654 ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
3655 ang_aux = ang_aux.reshape(1,ang_aux.size)
3655 ang_aux = ang_aux.reshape(1,ang_aux.size)
3656
3656
3657 cosdir = numpy.zeros((arrayPhase.shape[0],2))
3657 cosdir = numpy.zeros((arrayPhase.shape[0],2))
3658 cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
3658 cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
3659
3659
3660
3660
3661 for i in range(2):
3661 for i in range(2):
3662 ph0 = arrayPhase[:,pairsList[i][0]]
3662 ph0 = arrayPhase[:,pairsList[i][0]]
3663 ph1 = arrayPhase[:,pairsList[i][1]]
3663 ph1 = arrayPhase[:,pairsList[i][1]]
3664 d0 = distances[pairsList[i][0]]
3664 d0 = distances[pairsList[i][0]]
3665 d1 = distances[pairsList[i][1]]
3665 d1 = distances[pairsList[i][1]]
3666
3666
3667 ph0_aux = ph0 + ph1
3667 ph0_aux = ph0 + ph1
3668 ph0_aux = numpy.angle(numpy.exp(1j*ph0_aux))
3668 ph0_aux = numpy.angle(numpy.exp(1j*ph0_aux))
3669 # ph0_aux[ph0_aux > numpy.pi] -= 2*numpy.pi
3669 # ph0_aux[ph0_aux > numpy.pi] -= 2*numpy.pi
3670 # ph0_aux[ph0_aux < -numpy.pi] += 2*numpy.pi
3670 # ph0_aux[ph0_aux < -numpy.pi] += 2*numpy.pi
3671 #First Estimation
3671 #First Estimation
3672 cosdir0[:,i] = (ph0_aux)/(2*numpy.pi*(d0 - d1))
3672 cosdir0[:,i] = (ph0_aux)/(2*numpy.pi*(d0 - d1))
3673
3673
3674 #Most-Accurate Second Estimation
3674 #Most-Accurate Second Estimation
3675 phi1_aux = ph0 - ph1
3675 phi1_aux = ph0 - ph1
3676 phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
3676 phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
3677 #Direction Cosine 1
3677 #Direction Cosine 1
3678 cosdir1 = (phi1_aux + ang_aux)/(2*numpy.pi*(d0 + d1))
3678 cosdir1 = (phi1_aux + ang_aux)/(2*numpy.pi*(d0 + d1))
3679
3679
3680 #Searching the correct Direction Cosine
3680 #Searching the correct Direction Cosine
3681 cosdir0_aux = cosdir0[:,i]
3681 cosdir0_aux = cosdir0[:,i]
3682 cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
3682 cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
3683 #Minimum Distance
3683 #Minimum Distance
3684 cosDiff = (cosdir1 - cosdir0_aux)**2
3684 cosDiff = (cosdir1 - cosdir0_aux)**2
3685 indcos = cosDiff.argmin(axis = 1)
3685 indcos = cosDiff.argmin(axis = 1)
3686 #Saving Value obtained
3686 #Saving Value obtained
3687 cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
3687 cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
3688
3688
3689 return cosdir0, cosdir
3689 return cosdir0, cosdir
3690
3690
3691 def __calculateAOA(self, cosdir, azimuth):
3691 def __calculateAOA(self, cosdir, azimuth):
3692 cosdirX = cosdir[:,0]
3692 cosdirX = cosdir[:,0]
3693 cosdirY = cosdir[:,1]
3693 cosdirY = cosdir[:,1]
3694
3694
3695 zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
3695 zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
3696 azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth#0 deg north, 90 deg east
3696 azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth#0 deg north, 90 deg east
3697 angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
3697 angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
3698
3698
3699 return angles
3699 return angles
3700
3700
3701 def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
3701 def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
3702
3702
3703 Ramb = 375 #Ramb = c/(2*PRF)
3703 Ramb = 375 #Ramb = c/(2*PRF)
3704 Re = 6371 #Earth Radius
3704 Re = 6371 #Earth Radius
3705 heights = numpy.zeros(Ranges.shape)
3705 heights = numpy.zeros(Ranges.shape)
3706
3706
3707 R_aux = numpy.array([0,1,2])*Ramb
3707 R_aux = numpy.array([0,1,2])*Ramb
3708 R_aux = R_aux.reshape(1,R_aux.size)
3708 R_aux = R_aux.reshape(1,R_aux.size)
3709
3709
3710 Ranges = Ranges.reshape(Ranges.size,1)
3710 Ranges = Ranges.reshape(Ranges.size,1)
3711
3711
3712 Ri = Ranges + R_aux
3712 Ri = Ranges + R_aux
3713 hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
3713 hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
3714
3714
3715 #Check if there is a height between 70 and 110 km
3715 #Check if there is a height between 70 and 110 km
3716 h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
3716 h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
3717 ind_h = numpy.where(h_bool == 1)[0]
3717 ind_h = numpy.where(h_bool == 1)[0]
3718
3718
3719 hCorr = hi[ind_h, :]
3719 hCorr = hi[ind_h, :]
3720 ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
3720 ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
3721
3721
3722 hCorr = hi[ind_hCorr][:len(ind_h)]
3722 hCorr = hi[ind_hCorr][:len(ind_h)]
3723 heights[ind_h] = hCorr
3723 heights[ind_h] = hCorr
3724
3724
3725 #Setting Error
3725 #Setting Error
3726 #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
3726 #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
3727 #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
3727 #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
3728 indError = numpy.where(numpy.logical_or(error == 13, error == 14))[0]
3728 indError = numpy.where(numpy.logical_or(error == 13, error == 14))[0]
3729 error[indError] = 0
3729 error[indError] = 0
3730 indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
3730 indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
3731 error[indInvalid2] = 14
3731 error[indInvalid2] = 14
3732 indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
3732 indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
3733 error[indInvalid1] = 13
3733 error[indInvalid1] = 13
3734
3734
3735 return heights, error
3735 return heights, error
3736
3736
3737 def getPhasePairs(self, channelPositions):
3737 def getPhasePairs(self, channelPositions):
3738 chanPos = numpy.array(channelPositions)
3738 chanPos = numpy.array(channelPositions)
3739 listOper = list(itertools.combinations(list(range(5)),2))
3739 listOper = list(itertools.combinations(list(range(5)),2))
3740
3740
3741 distances = numpy.zeros(4)
3741 distances = numpy.zeros(4)
3742 axisX = []
3742 axisX = []
3743 axisY = []
3743 axisY = []
3744 distX = numpy.zeros(3)
3744 distX = numpy.zeros(3)
3745 distY = numpy.zeros(3)
3745 distY = numpy.zeros(3)
3746 ix = 0
3746 ix = 0
3747 iy = 0
3747 iy = 0
3748
3748
3749 pairX = numpy.zeros((2,2))
3749 pairX = numpy.zeros((2,2))
3750 pairY = numpy.zeros((2,2))
3750 pairY = numpy.zeros((2,2))
3751
3751
3752 for i in range(len(listOper)):
3752 for i in range(len(listOper)):
3753 pairi = listOper[i]
3753 pairi = listOper[i]
3754
3754
3755 posDif = numpy.abs(chanPos[pairi[0],:] - chanPos[pairi[1],:])
3755 posDif = numpy.abs(chanPos[pairi[0],:] - chanPos[pairi[1],:])
3756
3756
3757 if posDif[0] == 0:
3757 if posDif[0] == 0:
3758 axisY.append(pairi)
3758 axisY.append(pairi)
3759 distY[iy] = posDif[1]
3759 distY[iy] = posDif[1]
3760 iy += 1
3760 iy += 1
3761 elif posDif[1] == 0:
3761 elif posDif[1] == 0:
3762 axisX.append(pairi)
3762 axisX.append(pairi)
3763 distX[ix] = posDif[0]
3763 distX[ix] = posDif[0]
3764 ix += 1
3764 ix += 1
3765
3765
3766 for i in range(2):
3766 for i in range(2):
3767 if i==0:
3767 if i==0:
3768 dist0 = distX
3768 dist0 = distX
3769 axis0 = axisX
3769 axis0 = axisX
3770 else:
3770 else:
3771 dist0 = distY
3771 dist0 = distY
3772 axis0 = axisY
3772 axis0 = axisY
3773
3773
3774 side = numpy.argsort(dist0)[:-1]
3774 side = numpy.argsort(dist0)[:-1]
3775 axis0 = numpy.array(axis0)[side,:]
3775 axis0 = numpy.array(axis0)[side,:]
3776 chanC = int(numpy.intersect1d(axis0[0,:], axis0[1,:])[0])
3776 chanC = int(numpy.intersect1d(axis0[0,:], axis0[1,:])[0])
3777 axis1 = numpy.unique(numpy.reshape(axis0,4))
3777 axis1 = numpy.unique(numpy.reshape(axis0,4))
3778 side = axis1[axis1 != chanC]
3778 side = axis1[axis1 != chanC]
3779 diff1 = chanPos[chanC,i] - chanPos[side[0],i]
3779 diff1 = chanPos[chanC,i] - chanPos[side[0],i]
3780 diff2 = chanPos[chanC,i] - chanPos[side[1],i]
3780 diff2 = chanPos[chanC,i] - chanPos[side[1],i]
3781 if diff1<0:
3781 if diff1<0:
3782 chan2 = side[0]
3782 chan2 = side[0]
3783 d2 = numpy.abs(diff1)
3783 d2 = numpy.abs(diff1)
3784 chan1 = side[1]
3784 chan1 = side[1]
3785 d1 = numpy.abs(diff2)
3785 d1 = numpy.abs(diff2)
3786 else:
3786 else:
3787 chan2 = side[1]
3787 chan2 = side[1]
3788 d2 = numpy.abs(diff2)
3788 d2 = numpy.abs(diff2)
3789 chan1 = side[0]
3789 chan1 = side[0]
3790 d1 = numpy.abs(diff1)
3790 d1 = numpy.abs(diff1)
3791
3791
3792 if i==0:
3792 if i==0:
3793 chanCX = chanC
3793 chanCX = chanC
3794 chan1X = chan1
3794 chan1X = chan1
3795 chan2X = chan2
3795 chan2X = chan2
3796 distances[0:2] = numpy.array([d1,d2])
3796 distances[0:2] = numpy.array([d1,d2])
3797 else:
3797 else:
3798 chanCY = chanC
3798 chanCY = chanC
3799 chan1Y = chan1
3799 chan1Y = chan1
3800 chan2Y = chan2
3800 chan2Y = chan2
3801 distances[2:4] = numpy.array([d1,d2])
3801 distances[2:4] = numpy.array([d1,d2])
3802 # axisXsides = numpy.reshape(axisX[ix,:],4)
3802 # axisXsides = numpy.reshape(axisX[ix,:],4)
3803 #
3803 #
3804 # channelCentX = int(numpy.intersect1d(pairX[0,:], pairX[1,:])[0])
3804 # channelCentX = int(numpy.intersect1d(pairX[0,:], pairX[1,:])[0])
3805 # channelCentY = int(numpy.intersect1d(pairY[0,:], pairY[1,:])[0])
3805 # channelCentY = int(numpy.intersect1d(pairY[0,:], pairY[1,:])[0])
3806 #
3806 #
3807 # ind25X = numpy.where(pairX[0,:] != channelCentX)[0][0]
3807 # ind25X = numpy.where(pairX[0,:] != channelCentX)[0][0]
3808 # ind20X = numpy.where(pairX[1,:] != channelCentX)[0][0]
3808 # ind20X = numpy.where(pairX[1,:] != channelCentX)[0][0]
3809 # channel25X = int(pairX[0,ind25X])
3809 # channel25X = int(pairX[0,ind25X])
3810 # channel20X = int(pairX[1,ind20X])
3810 # channel20X = int(pairX[1,ind20X])
3811 # ind25Y = numpy.where(pairY[0,:] != channelCentY)[0][0]
3811 # ind25Y = numpy.where(pairY[0,:] != channelCentY)[0][0]
3812 # ind20Y = numpy.where(pairY[1,:] != channelCentY)[0][0]
3812 # ind20Y = numpy.where(pairY[1,:] != channelCentY)[0][0]
3813 # channel25Y = int(pairY[0,ind25Y])
3813 # channel25Y = int(pairY[0,ind25Y])
3814 # channel20Y = int(pairY[1,ind20Y])
3814 # channel20Y = int(pairY[1,ind20Y])
3815
3815
3816 # pairslist = [(channelCentX, channel25X),(channelCentX, channel20X),(channelCentY,channel25Y),(channelCentY, channel20Y)]
3816 # pairslist = [(channelCentX, channel25X),(channelCentX, channel20X),(channelCentY,channel25Y),(channelCentY, channel20Y)]
3817 pairslist = [(chanCX, chan1X),(chanCX, chan2X),(chanCY,chan1Y),(chanCY, chan2Y)]
3817 pairslist = [(chanCX, chan1X),(chanCX, chan2X),(chanCY,chan1Y),(chanCY, chan2Y)]
3818
3818
3819 return pairslist, distances
3819 return pairslist, distances
3820 # def __getAOA(self, phases, pairsList, error, AOAthresh, azimuth):
3820 # def __getAOA(self, phases, pairsList, error, AOAthresh, azimuth):
3821 #
3821 #
3822 # arrayAOA = numpy.zeros((phases.shape[0],3))
3822 # arrayAOA = numpy.zeros((phases.shape[0],3))
3823 # cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList)
3823 # cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList)
3824 #
3824 #
3825 # arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
3825 # arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
3826 # cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
3826 # cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
3827 # arrayAOA[:,2] = cosDirError
3827 # arrayAOA[:,2] = cosDirError
3828 #
3828 #
3829 # azimuthAngle = arrayAOA[:,0]
3829 # azimuthAngle = arrayAOA[:,0]
3830 # zenithAngle = arrayAOA[:,1]
3830 # zenithAngle = arrayAOA[:,1]
3831 #
3831 #
3832 # #Setting Error
3832 # #Setting Error
3833 # #Number 3: AOA not fesible
3833 # #Number 3: AOA not fesible
3834 # indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
3834 # indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
3835 # error[indInvalid] = 3
3835 # error[indInvalid] = 3
3836 # #Number 4: Large difference in AOAs obtained from different antenna baselines
3836 # #Number 4: Large difference in AOAs obtained from different antenna baselines
3837 # indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
3837 # indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
3838 # error[indInvalid] = 4
3838 # error[indInvalid] = 4
3839 # return arrayAOA, error
3839 # return arrayAOA, error
3840 #
3840 #
3841 # def __getDirectionCosines(self, arrayPhase, pairsList):
3841 # def __getDirectionCosines(self, arrayPhase, pairsList):
3842 #
3842 #
3843 # #Initializing some variables
3843 # #Initializing some variables
3844 # ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
3844 # ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
3845 # ang_aux = ang_aux.reshape(1,ang_aux.size)
3845 # ang_aux = ang_aux.reshape(1,ang_aux.size)
3846 #
3846 #
3847 # cosdir = numpy.zeros((arrayPhase.shape[0],2))
3847 # cosdir = numpy.zeros((arrayPhase.shape[0],2))
3848 # cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
3848 # cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
3849 #
3849 #
3850 #
3850 #
3851 # for i in range(2):
3851 # for i in range(2):
3852 # #First Estimation
3852 # #First Estimation
3853 # phi0_aux = arrayPhase[:,pairsList[i][0]] + arrayPhase[:,pairsList[i][1]]
3853 # phi0_aux = arrayPhase[:,pairsList[i][0]] + arrayPhase[:,pairsList[i][1]]
3854 # #Dealias
3854 # #Dealias
3855 # indcsi = numpy.where(phi0_aux > numpy.pi)
3855 # indcsi = numpy.where(phi0_aux > numpy.pi)
3856 # phi0_aux[indcsi] -= 2*numpy.pi
3856 # phi0_aux[indcsi] -= 2*numpy.pi
3857 # indcsi = numpy.where(phi0_aux < -numpy.pi)
3857 # indcsi = numpy.where(phi0_aux < -numpy.pi)
3858 # phi0_aux[indcsi] += 2*numpy.pi
3858 # phi0_aux[indcsi] += 2*numpy.pi
3859 # #Direction Cosine 0
3859 # #Direction Cosine 0
3860 # cosdir0[:,i] = -(phi0_aux)/(2*numpy.pi*0.5)
3860 # cosdir0[:,i] = -(phi0_aux)/(2*numpy.pi*0.5)
3861 #
3861 #
3862 # #Most-Accurate Second Estimation
3862 # #Most-Accurate Second Estimation
3863 # phi1_aux = arrayPhase[:,pairsList[i][0]] - arrayPhase[:,pairsList[i][1]]
3863 # phi1_aux = arrayPhase[:,pairsList[i][0]] - arrayPhase[:,pairsList[i][1]]
3864 # phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
3864 # phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
3865 # #Direction Cosine 1
3865 # #Direction Cosine 1
3866 # cosdir1 = -(phi1_aux + ang_aux)/(2*numpy.pi*4.5)
3866 # cosdir1 = -(phi1_aux + ang_aux)/(2*numpy.pi*4.5)
3867 #
3867 #
3868 # #Searching the correct Direction Cosine
3868 # #Searching the correct Direction Cosine
3869 # cosdir0_aux = cosdir0[:,i]
3869 # cosdir0_aux = cosdir0[:,i]
3870 # cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
3870 # cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
3871 # #Minimum Distance
3871 # #Minimum Distance
3872 # cosDiff = (cosdir1 - cosdir0_aux)**2
3872 # cosDiff = (cosdir1 - cosdir0_aux)**2
3873 # indcos = cosDiff.argmin(axis = 1)
3873 # indcos = cosDiff.argmin(axis = 1)
3874 # #Saving Value obtained
3874 # #Saving Value obtained
3875 # cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
3875 # cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
3876 #
3876 #
3877 # return cosdir0, cosdir
3877 # return cosdir0, cosdir
3878 #
3878 #
3879 # def __calculateAOA(self, cosdir, azimuth):
3879 # def __calculateAOA(self, cosdir, azimuth):
3880 # cosdirX = cosdir[:,0]
3880 # cosdirX = cosdir[:,0]
3881 # cosdirY = cosdir[:,1]
3881 # cosdirY = cosdir[:,1]
3882 #
3882 #
3883 # zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
3883 # zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
3884 # azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth #0 deg north, 90 deg east
3884 # azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth #0 deg north, 90 deg east
3885 # angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
3885 # angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
3886 #
3886 #
3887 # return angles
3887 # return angles
3888 #
3888 #
3889 # def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
3889 # def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
3890 #
3890 #
3891 # Ramb = 375 #Ramb = c/(2*PRF)
3891 # Ramb = 375 #Ramb = c/(2*PRF)
3892 # Re = 6371 #Earth Radius
3892 # Re = 6371 #Earth Radius
3893 # heights = numpy.zeros(Ranges.shape)
3893 # heights = numpy.zeros(Ranges.shape)
3894 #
3894 #
3895 # R_aux = numpy.array([0,1,2])*Ramb
3895 # R_aux = numpy.array([0,1,2])*Ramb
3896 # R_aux = R_aux.reshape(1,R_aux.size)
3896 # R_aux = R_aux.reshape(1,R_aux.size)
3897 #
3897 #
3898 # Ranges = Ranges.reshape(Ranges.size,1)
3898 # Ranges = Ranges.reshape(Ranges.size,1)
3899 #
3899 #
3900 # Ri = Ranges + R_aux
3900 # Ri = Ranges + R_aux
3901 # hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
3901 # hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
3902 #
3902 #
3903 # #Check if there is a height between 70 and 110 km
3903 # #Check if there is a height between 70 and 110 km
3904 # h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
3904 # h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
3905 # ind_h = numpy.where(h_bool == 1)[0]
3905 # ind_h = numpy.where(h_bool == 1)[0]
3906 #
3906 #
3907 # hCorr = hi[ind_h, :]
3907 # hCorr = hi[ind_h, :]
3908 # ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
3908 # ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
3909 #
3909 #
3910 # hCorr = hi[ind_hCorr]
3910 # hCorr = hi[ind_hCorr]
3911 # heights[ind_h] = hCorr
3911 # heights[ind_h] = hCorr
3912 #
3912 #
3913 # #Setting Error
3913 # #Setting Error
3914 # #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
3914 # #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
3915 # #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
3915 # #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
3916 #
3916 #
3917 # indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
3917 # indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
3918 # error[indInvalid2] = 14
3918 # error[indInvalid2] = 14
3919 # indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
3919 # indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
3920 # error[indInvalid1] = 13
3920 # error[indInvalid1] = 13
3921 #
3921 #
3922 # return heights, error
3922 # return heights, error
3923
3923
3924
3924
3925 class WeatherRadar(Operation):
3925 class WeatherRadar(Operation):
3926 '''
3926 '''
3927 Function tat implements Weather Radar operations-
3927 Function tat implements Weather Radar operations-
3928 Input:
3928 Input:
3929 Output:
3929 Output:
3930 Parameters affected:
3930 Parameters affected:
3931
3931
3932 Conversion Watt
3932 Conversion Watt
3933 Referencia
3933 Referencia
3934 https://www.tek.com/en/blog/calculating-rf-power-iq-samples
3934 https://www.tek.com/en/blog/calculating-rf-power-iq-samples
3935
3935
3936 data_param = (nCh, 8, nHeis)
3936 data_param = (nCh, 8, nHeis)
3937 S, V, W, SNR, Z, D, P, R
3937 S, V, W, SNR, Z, D, P, R
3938 Power, Velocity, Spectral width, SNR, Reflectivity, Differential reflectivity, PHI DP, RHO HV
3938 Power, Velocity, Spectral width, SNR, Reflectivity, Differential reflectivity, PHI DP, RHO HV
3939 '''
3939 '''
3940 isConfig = False
3940 isConfig = False
3941 variableList = None
3941 variableList = None
3942
3942
3943 def __init__(self):
3943 def __init__(self):
3944 Operation.__init__(self)
3944 Operation.__init__(self)
3945
3945
3946 def setup(self,dataOut,variableList= None,Pt=0,Gt=0,Gr=0,Glna=0,lambda_=0, aL=0,
3946 def setup(self,dataOut,variableList= None,Pt=0,Gt=0,Gr=0,Glna=0,lambda_=0, aL=0,
3947 tauW= 0,thetaT=0,thetaR=0,Km =0):
3947 tauW= 0,thetaT=0,thetaR=0,Km =0):
3948
3948
3949 self.nCh = dataOut.nChannels
3949 self.nCh = dataOut.nChannels
3950 self.nHeis = dataOut.nHeights
3950 self.nHeis = dataOut.nHeights
3951 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
3951 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
3952 self.Range = numpy.arange(dataOut.nHeights)*deltaHeight + dataOut.heightList[0]
3952 self.Range = numpy.arange(dataOut.nHeights)*deltaHeight + dataOut.heightList[0]
3953 self.Range = self.Range.reshape(1,self.nHeis)
3953 self.Range = self.Range.reshape(1,self.nHeis)
3954 self.Range = numpy.tile(self.Range,[self.nCh,1])
3954 self.Range = numpy.tile(self.Range,[self.nCh,1])
3955 '''-----------1 Constante del Radar----------'''
3955 '''-----------1 Constante del Radar----------'''
3956 self.Pt = Pt # Pmax =200 W x DC=(0.2 useg/400useg)
3956 self.Pt = Pt # Pmax =200 W x DC=(0.2 useg/400useg)
3957 self.Gt = Gt # 38 db
3957 self.Gt = Gt # 38 db
3958 self.Gr = Gr # 38 dB
3958 self.Gr = Gr # 38 dB
3959 self.Glna = Glna # 60 dB
3959 self.Glna = Glna # 60 dB
3960 self.lambda_ = lambda_ # 3.2 cm 0.032 m.
3960 self.lambda_ = lambda_ # 3.2 cm 0.032 m.
3961 self.aL = aL # Perdidas
3961 self.aL = aL # Perdidas
3962 self.tauW = tauW #ancho de pulso 0.2useg pulso corto.
3962 self.tauW = tauW #ancho de pulso 0.2useg pulso corto.
3963 self.thetaT = thetaT # 1.8º -- 0.0314 rad
3963 self.thetaT = thetaT # 1.8º -- 0.0314 rad
3964 self.thetaR = thetaR # 1.8ª --0.0314 rad
3964 self.thetaR = thetaR # 1.8ª --0.0314 rad
3965 self.Km = Km
3965 self.Km = Km
3966 Numerator = ((4*numpy.pi)**3 * aL**2 * 16 *numpy.log(2)*(10**18))
3966 Numerator = ((4*numpy.pi)**3 * aL**2 * 16 *numpy.log(2)*(10**18))
3967 Denominator = (Pt *(10**(Gt/10.0))*(10**(Gr/10.0))*(10**(Glna/10.0))* lambda_**2 * SPEED_OF_LIGHT * tauW * numpy.pi*thetaT*thetaR)
3967 Denominator = (Pt *(10**(Gt/10.0))*(10**(Gr/10.0))*(10**(Glna/10.0))* lambda_**2 * SPEED_OF_LIGHT * tauW * numpy.pi*thetaT*thetaR)
3968 self.RadarConstant = Numerator/Denominator
3968 self.RadarConstant = Numerator/Denominator
3969 self.variableList = variableList
3969 self.variableList = variableList
3970 if self.variableList== None:
3970 if self.variableList== None:
3971 self.variableList= ['Z','D','R','P']
3971 self.variableList= ['Z','D','R','P']
3972
3972
3973 def setMoments(self, dataOut):
3973 def setMoments(self, dataOut):
3974 # S, V, W, SNR, Z, D, P, R
3974 # S, V, W, SNR, Z, D, P, R
3975 type = dataOut.inputUnit
3975 type = dataOut.inputUnit
3976 nCh = dataOut.nChannels
3976 nCh = dataOut.nChannels
3977 nHeis = dataOut.nHeights
3977 nHeis = dataOut.nHeights
3978 data_param = numpy.zeros((nCh, 8, nHeis))
3978 data_param = numpy.zeros((nCh, 8, nHeis))
3979 if type == "Voltage":
3979 if type == "Voltage":
3980 factor = 1
3980 factor = 1
3981 data_param[:,0,:] = dataOut.dataPP_POW/(factor)#dataOut.dataPP_POWER/(factor)
3981 data_param[:,0,:] = dataOut.dataPP_POW/(factor)#dataOut.dataPP_POWER/(factor)
3982 data_param[:,1,:] = dataOut.dataPP_DOP
3982 data_param[:,1,:] = dataOut.dataPP_DOP
3983 data_param[:,2,:] = dataOut.dataPP_WIDTH
3983 data_param[:,2,:] = dataOut.dataPP_WIDTH
3984 data_param[:,3,:] = dataOut.dataPP_SNR
3984 data_param[:,3,:] = dataOut.dataPP_SNR
3985 if type == "Spectra":
3985 if type == "Spectra":
3986 factor = dataOut.normFactor
3986 factor = dataOut.normFactor
3987 data_param[:,0,:] = dataOut.data_pow/(factor)
3987 data_param[:,0,:] = dataOut.data_pow/(factor)
3988 data_param[:,1,:] = dataOut.data_dop
3988 data_param[:,1,:] = dataOut.data_dop
3989 data_param[:,2,:] = dataOut.data_width
3989 data_param[:,2,:] = dataOut.data_width
3990 data_param[:,3,:] = dataOut.data_snr
3990 data_param[:,3,:] = dataOut.data_snr
3991 return data_param
3991 return data_param
3992
3992
3993 def getCoeficienteCorrelacionROhv_R(self,dataOut):
3993 def getCoeficienteCorrelacionROhv_R(self,dataOut):
3994 type = dataOut.inputUnit
3994 type = dataOut.inputUnit
3995 nHeis = dataOut.nHeights
3995 nHeis = dataOut.nHeights
3996 data_RhoHV_R = numpy.zeros((nHeis))
3996 data_RhoHV_R = numpy.zeros((nHeis))
3997 if type == "Voltage":
3997 if type == "Voltage":
3998 powa = dataOut.data_param[0,0,:]
3998 powa = dataOut.data_param[0,0,:]
3999 powb = dataOut.data_param[1,0,:]
3999 powb = dataOut.data_param[1,0,:]
4000 ccf = dataOut.dataPP_CCF
4000 ccf = dataOut.dataPP_CCF
4001 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
4001 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
4002 data_RhoHV_R = numpy.abs(avgcoherenceComplex)
4002 data_RhoHV_R = numpy.abs(avgcoherenceComplex)
4003 if type == "Spectra":
4003 if type == "Spectra":
4004 data_RhoHV_R = dataOut.getCoherence()
4004 data_RhoHV_R = dataOut.getCoherence()
4005
4005
4006 return data_RhoHV_R
4006 return data_RhoHV_R
4007
4007
4008 def getFasediferencialPhiD_P(self,dataOut,phase= True):
4008 def getFasediferencialPhiD_P(self,dataOut,phase= True):
4009 type = dataOut.inputUnit
4009 type = dataOut.inputUnit
4010 nHeis = dataOut.nHeights
4010 nHeis = dataOut.nHeights
4011 data_PhiD_P = numpy.zeros((nHeis))
4011 data_PhiD_P = numpy.zeros((nHeis))
4012 if type == "Voltage":
4012 if type == "Voltage":
4013 powa = dataOut.data_param[0,0,:]
4013 powa = dataOut.data_param[0,0,:]
4014 powb = dataOut.data_param[1,0,:]
4014 powb = dataOut.data_param[1,0,:]
4015 ccf = dataOut.dataPP_CCF
4015 ccf = dataOut.dataPP_CCF
4016 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
4016 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
4017 if phase:
4017 if phase:
4018 data_PhiD_P = numpy.arctan2(avgcoherenceComplex.imag,
4018 data_PhiD_P = numpy.arctan2(avgcoherenceComplex.imag,
4019 avgcoherenceComplex.real) * 180 / numpy.pi
4019 avgcoherenceComplex.real) * 180 / numpy.pi
4020 if type == "Spectra":
4020 if type == "Spectra":
4021 data_PhiD_P = dataOut.getCoherence(phase = phase)
4021 data_PhiD_P = dataOut.getCoherence(phase = phase)
4022
4022
4023 return data_PhiD_P
4023 return data_PhiD_P
4024
4024
4025 def getReflectividad_D(self,dataOut,type):
4025 def getReflectividad_D(self,dataOut,type):
4026 '''-----------------------------Potencia de Radar -Signal S-----------------------------'''
4026 '''-----------------------------Potencia de Radar -Signal S-----------------------------'''
4027
4027
4028 Pr = dataOut.data_param[:,0,:]
4028 Pr = dataOut.data_param[:,0,:]
4029 '''---------------------------- Calculo de Noise y threshold para Reflectividad---------'''
4029 '''---------------------------- Calculo de Noise y threshold para Reflectividad---------'''
4030 # noise = numpy.zeros(self.nCh)
4030 # noise = numpy.zeros(self.nCh)
4031 # for i in range(self.nCh):
4031 # for i in range(self.nCh):
4032 # noise[i] = hildebrand_sekhon(Pr[i,:], 1)
4032 # noise[i] = hildebrand_sekhon(Pr[i,:], 1)
4033 # window = numpy.where(Pr[i,:]<1.3*noise[i])
4033 # window = numpy.where(Pr[i,:]<1.3*noise[i])
4034 # Pr[i,window]= 1e-10
4034 # Pr[i,window]= 1e-10
4035 Pr = Pr/1000.0 # Conversion Watt
4035 Pr = Pr/1000.0 # Conversion Watt
4036 '''-----------2 Reflectividad del Radar y Factor de Reflectividad------'''
4036 '''-----------2 Reflectividad del Radar y Factor de Reflectividad------'''
4037 self.n_radar = numpy.zeros((self.nCh,self.nHeis))
4037 self.n_radar = numpy.zeros((self.nCh,self.nHeis))
4038 self.Z_radar = numpy.zeros((self.nCh,self.nHeis))
4038 self.Z_radar = numpy.zeros((self.nCh,self.nHeis))
4039
4039
4040 for R in range(self.nHeis):
4040 for R in range(self.nHeis):
4041 self.n_radar[:,R] = self.RadarConstant*Pr[:,R]* (self.Range[:,R]*(10**3))**2
4041 self.n_radar[:,R] = self.RadarConstant*Pr[:,R]* (self.Range[:,R]*(10**3))**2
4042
4042
4043 self.Z_radar[:,R] = self.n_radar[:,R]* self.lambda_**4/( numpy.pi**5 * self.Km**2)
4043 self.Z_radar[:,R] = self.n_radar[:,R]* self.lambda_**4/( numpy.pi**5 * self.Km**2)
4044
4044
4045 '''----------- Factor de Reflectividad Equivalente lamda_ < 10 cm , lamda_= 3.2cm-------'''
4045 '''----------- Factor de Reflectividad Equivalente lamda_ < 10 cm , lamda_= 3.2cm-------'''
4046 Zeh = self.Z_radar
4046 Zeh = self.Z_radar
4047 #print("---------------------------------------------------------------------")
4047 #print("---------------------------------------------------------------------")
4048 #print("RangedBz",10*numpy.log10((self.Range[0,-10:]*(10**3))**2))
4048 #print("RangedBz",10*numpy.log10((self.Range[0,-10:]*(10**3))**2))
4049 #print("CTE",10*numpy.log10(self.RadarConstant))
4049 #print("CTE",10*numpy.log10(self.RadarConstant))
4050 #print("Pr first10",10*numpy.log10(Pr[0,:20]))
4050 #print("Pr first10",10*numpy.log10(Pr[0,:20]))
4051 #print("Pr last10",10*numpy.log10(Pr[0,-20:]))
4051 #print("Pr last10",10*numpy.log10(Pr[0,-20:]))
4052 #print("LCTE",10*numpy.log10(self.lambda_**4/( numpy.pi**5 * self.Km**2)))
4052 #print("LCTE",10*numpy.log10(self.lambda_**4/( numpy.pi**5 * self.Km**2)))
4053 if self.Pt<0.3:
4053 if self.Pt<0.3:
4054 factor=10.072
4054 factor=10.072
4055 else:
4055 else:
4056 factor=23.072
4056 factor=23.072
4057
4057
4058 dBZeh = 10*numpy.log10(Zeh) + factor
4058 dBZeh = 10*numpy.log10(Zeh) + factor
4059 if type=='N':
4059 if type=='N':
4060 return dBZeh
4060 return dBZeh
4061 elif type=='D':
4061 elif type=='D':
4062 Zdb_D = dBZeh[0] - dBZeh[1]
4062 Zdb_D = dBZeh[0] - dBZeh[1]
4063 return Zdb_D
4063 return Zdb_D
4064
4064
4065 def getRadialVelocity_V(self,dataOut):
4065 def getRadialVelocity_V(self,dataOut):
4066 velRadial_V = dataOut.data_param[:,1,:]
4066 velRadial_V = dataOut.data_param[:,1,:]
4067 return velRadial_V
4067 return velRadial_V
4068
4068
4069 def getAnchoEspectral_W(self,dataOut):
4069 def getAnchoEspectral_W(self,dataOut):
4070 Sigmav_W = dataOut.data_param[:,2,:]
4070 Sigmav_W = dataOut.data_param[:,2,:]
4071 return Sigmav_W
4071 return Sigmav_W
4072
4072
4073
4073
4074 def run(self,dataOut,variableList=None,Pt=1.58,Gt=38.5,Gr=38.5,Glna=70.0,lambda_=0.032, aL=1,
4074 def run(self,dataOut,variableList=None,Pt=1.58,Gt=38.5,Gr=38.5,Glna=70.0,lambda_=0.032, aL=1,
4075 tauW= 0.2,thetaT=0.0314,thetaR=0.0314,Km =0.93):
4075 tauW= 0.2,thetaT=0.0314,thetaR=0.0314,Km =0.93):
4076
4076
4077 if not self.isConfig:
4077 if not self.isConfig:
4078 self.setup(dataOut= dataOut, variableList=variableList,Pt=Pt,Gt=Gt,Gr=Gr,Glna=Glna,lambda_=lambda_, aL=aL,
4078 self.setup(dataOut= dataOut, variableList=variableList,Pt=Pt,Gt=Gt,Gr=Gr,Glna=Glna,lambda_=lambda_, aL=aL,
4079 tauW= tauW,thetaT=thetaT,thetaR=thetaR,Km =Km)
4079 tauW= tauW,thetaT=thetaT,thetaR=thetaR,Km =Km)
4080 self.isConfig = True
4080 self.isConfig = True
4081
4081
4082 dataOut.data_param = self.setMoments(dataOut)
4082 dataOut.data_param = self.setMoments(dataOut)
4083
4083
4084 for i in range(len(self.variableList)):
4084 for i in range(len(self.variableList)):
4085 if self.variableList[i] == 'Z':
4085 if self.variableList[i] == 'Z':
4086 dataOut.data_param[:,4,:] =self.getReflectividad_D(dataOut=dataOut,type='N')
4086 dataOut.data_param[:,4,:] =self.getReflectividad_D(dataOut=dataOut,type='N')
4087 if self.variableList[i] == 'D' and dataOut.nChannels>1:
4087 if self.variableList[i] == 'D' and dataOut.nChannels>1:
4088 dataOut.data_param[:,5,:] =self.getReflectividad_D(dataOut=dataOut,type='D')
4088 dataOut.data_param[:,5,:] =self.getReflectividad_D(dataOut=dataOut,type='D')
4089 if self.variableList[i] == 'P' and dataOut.nChannels>1:
4089 if self.variableList[i] == 'P' and dataOut.nChannels>1:
4090 dataOut.data_param[:,6,:] =self.getFasediferencialPhiD_P(dataOut=dataOut, phase=True)
4090 dataOut.data_param[:,6,:] =self.getFasediferencialPhiD_P(dataOut=dataOut, phase=True)
4091 if self.variableList[i] == 'R' and dataOut.nChannels>1:
4091 if self.variableList[i] == 'R' and dataOut.nChannels>1:
4092 dataOut.data_param[:,7,:] = self.getCoeficienteCorrelacionROhv_R(dataOut)
4092 dataOut.data_param[:,7,:] = self.getCoeficienteCorrelacionROhv_R(dataOut)
4093
4093
4094 return dataOut
4094 return dataOut
4095
4095
4096 class PedestalInformation(Operation):
4096 class PedestalInformation(Operation):
4097
4097
4098 def __init__(self):
4098 def __init__(self):
4099 Operation.__init__(self)
4099 Operation.__init__(self)
4100 self.filename = False
4100 self.filename = False
4101 self.delay = 32
4101 self.delay = 32
4102 self.nTries = 3
4102 self.nTries = 3
4103 self.nFiles = 5
4103 self.nFiles = 5
4104 self.flagAskMode = False
4104 self.flagAskMode = False
4105
4105
4106 def find_file(self, timestamp):
4106 def find_file(self, timestamp):
4107
4107
4108 dt = datetime.datetime.utcfromtimestamp(timestamp)
4108 dt = datetime.datetime.utcfromtimestamp(timestamp)
4109 path = os.path.join(self.path, dt.strftime('%Y-%m-%dT%H-00-00'))
4109 path = os.path.join(self.path, dt.strftime('%Y-%m-%dT%H-00-00'))
4110
4110
4111 if not os.path.exists(path):
4111 if not os.path.exists(path):
4112 return False
4112 return False
4113 fileList = glob.glob(os.path.join(path, '*.h5'))
4113 fileList = glob.glob(os.path.join(path, '*.h5'))
4114 fileList.sort()
4114 fileList.sort()
4115 return fileList
4115 return fileList
4116
4116
4117 def find_next_file(self):
4117 def find_next_file(self):
4118
4118
4119 while True:
4119 while True:
4120 if self.utctime < self.utcfile:
4120 if self.utctime < self.utcfile:
4121 self.flagNoData = True
4121 self.flagNoData = True
4122 break
4122 break
4123 self.flagNoData = False
4123 self.flagNoData = False
4124 file_size = len(self.fp['Data']['utc'])
4124 file_size = len(self.fp['Data']['utc'])
4125 if self.utctime < self.utcfile+file_size*self.interval:
4125 if self.utctime < self.utcfile+file_size*self.interval:
4126 break
4126 break
4127 dt = datetime.datetime.utcfromtimestamp(self.utcfile)
4127 dt = datetime.datetime.utcfromtimestamp(self.utcfile)
4128 if dt.second > 0:
4128 if dt.second > 0:
4129 self.utcfile -= dt.second
4129 self.utcfile -= dt.second
4130 self.utcfile += self.samples*self.interval
4130 self.utcfile += self.samples*self.interval
4131 dt = datetime.datetime.utcfromtimestamp(self.utcfile)
4131 dt = datetime.datetime.utcfromtimestamp(self.utcfile)
4132 path = os.path.join(self.path, dt.strftime('%Y-%m-%dT%H-00-00'))
4132 path = os.path.join(self.path, dt.strftime('%Y-%m-%dT%H-00-00'))
4133 self.filename = os.path.join(path, 'pos@{}.000.h5'.format(int(self.utcfile)))
4133 self.filename = os.path.join(path, 'pos@{}.000.h5'.format(int(self.utcfile)))
4134
4134
4135 for i in range(self.nFiles):
4135 for i in range(self.nFiles):
4136 ok = False
4136 ok = False
4137 for j in range(self.nTries):
4137 for j in range(self.nTries):
4138 ok = False
4138 ok = False
4139 try:
4139 try:
4140 if not os.path.exists(self.filename):
4140 if not os.path.exists(self.filename):
4141 log.warning('Waiting {}s for position files...'.format(self.delay), self.name)
4141 log.warning('Waiting {}s for position files...'.format(self.delay), self.name)
4142 time.sleep(1)
4142 time.sleep(1)
4143 continue
4143 continue
4144 self.fp.close()
4144 self.fp.close()
4145 self.fp = h5py.File(self.filename, 'r')
4145 self.fp = h5py.File(self.filename, 'r')
4146 self.ele = self.fp['Data']['ele_pos'][:]
4147 self.azi = self.fp['Data']['azi_pos'][:] + 26.27
4148 self.azi[self.azi>360] = self.azi[self.azi>360] - 360
4146 log.log('Opening file: {}'.format(self.filename), self.name)
4149 log.log('Opening file: {}'.format(self.filename), self.name)
4147 ok = True
4150 ok = True
4148 break
4151 break
4149 except Exception as e:
4152 except Exception as e:
4150 log.warning('Waiting {}s for position file to be ready...'.format(self.delay), self.name)
4153 log.warning('Waiting {}s for position file to be ready...'.format(self.delay), self.name)
4151 time.sleep(self.delay)
4154 time.sleep(self.delay)
4152 continue
4155 continue
4153 if ok:
4156 if ok:
4154 break
4157 break
4155 log.warning('Trying next file...', self.name)
4158 log.warning('Trying next file...', self.name)
4156 self.utcfile += self.samples*self.interval
4159 self.utcfile += self.samples*self.interval
4157 dt = datetime.datetime.utcfromtimestamp(self.utcfile)
4160 dt = datetime.datetime.utcfromtimestamp(self.utcfile)
4158 path = os.path.join(self.path, dt.strftime('%Y-%m-%dT%H-00-00'))
4161 path = os.path.join(self.path, dt.strftime('%Y-%m-%dT%H-00-00'))
4159 self.filename = os.path.join(path, 'pos@{}.000.h5'.format(int(self.utcfile)))
4162 self.filename = os.path.join(path, 'pos@{}.000.h5'.format(int(self.utcfile)))
4160 if not ok:
4163 if not ok:
4161 log.error('No new position files found in {}'.format(path))
4164 log.error('No new position files found in {}'.format(path))
4162 raise IOError('No new position files found in {}'.format(path))
4165 raise IOError('No new position files found in {}'.format(path))
4163
4166
4164
4167
4165 def find_mode(self,index):
4168 def find_mode(self, index):
4166 sample_max = 20
4169 sample_max = 20
4167 start = index
4170 start = index
4168 flag_mode = None
4171 flag_mode = None
4169 azi = self.fp['Data']['azi_pos'][:]
4170 ele = self.fp['Data']['ele_pos'][:]
4171
4172
4172 while True:
4173 while True:
4173 if start+sample_max > numpy.shape(ele)[0]:
4174 print(start, sample_max, numpy.shape(self.ele))
4174 print("CANNOT KNOW IF MODE IS PPI OR RHI, ANALIZE NEXT FILE")
4175 if start+sample_max > numpy.shape(self.ele)[0]:
4175 print("ele",ele[start-sample_max:start+sample_max])
4176 print("azi",azi[start-sample_max:start+sample_max])
4177 if sample_max == 10:
4176 if sample_max == 10:
4177 print("CANNOT KNOW IF MODE IS PPI OR RHI, ANALIZE NEXT FILE")
4178 break
4178 break
4179 else:
4179 else:
4180 sample_max = 10
4180 sample_max = 10
4181 continue
4181 continue
4182 sigma_ele = numpy.nanstd(ele[start:start+sample_max])
4182 sigma_ele = numpy.nanstd(self.ele[start:start+sample_max])
4183 sigma_azi = numpy.nanstd(azi[start:start+sample_max])
4183 sigma_azi = numpy.nanstd(self.azi[start:start+sample_max])
4184 print("ele",self.ele[start-sample_max:start+sample_max])
4185 print("azi",self.azi[start-sample_max:start+sample_max])
4186 print(sigma_azi, sigma_ele)
4184
4187
4185 if sigma_ele<.5 and sigma_azi<.5:
4188 if sigma_ele<.5 and sigma_azi<.5:
4186 if sigma_ele<sigma_azi:
4189 if sigma_ele<sigma_azi:
4187 flag_mode = 'PPI'
4190 flag_mode = 'PPI'
4188 break
4191 break
4189 else:
4192 else:
4190 flag_mode = 'RHI'
4193 flag_mode = 'RHI'
4191 break
4194 break
4192 elif sigma_ele < .5:
4195 elif sigma_ele < .5:
4193 flag_mode = 'PPI'
4196 flag_mode = 'PPI'
4194 break
4197 break
4195 elif sigma_azi < .5:
4198 elif sigma_azi < .5:
4196 flag_mode = 'RHI'
4199 flag_mode = 'RHI'
4197 break
4200 break
4198
4201
4199 start += sample_max
4202 start += sample_max
4200
4203
4201 return flag_mode
4204 return flag_mode
4202
4205
4203 def get_values(self):
4206 def get_values(self):
4204
4207
4205 if self.flagNoData:
4208 if self.flagNoData:
4206 return numpy.nan, numpy.nan, numpy.nan #Should be self.mode?
4209 return numpy.nan, numpy.nan, numpy.nan #Should be self.mode?
4207 else:
4210 else:
4208 index = int((self.utctime-self.utcfile)/self.interval)
4211 index = int((self.utctime-self.utcfile)/self.interval)
4212 try:
4213 #print( self.azi[index], self.ele[index], None)
4214 return self.azi[index], self.ele[index], None
4215 except:
4216 return numpy.nan, numpy.nan, numpy.nan
4209
4217
4210 if self.flagAskMode:
4218 if self.flagAskMode:
4211 mode = self.find_mode(index)
4219 mode = self.find_mode(index)
4220 print('MODE: ', mode)
4212 else:
4221 else:
4213 mode = self.mode
4222 mode = self.mode
4214
4223
4215 if mode is not None:
4224 if mode is not None:
4216 return self.fp['Data']['azi_pos'][index], self.fp['Data']['ele_pos'][index], mode
4225 return self.azi[index], self.ele[index], mode
4217 else:
4226 else:
4218 return numpy.nan, numpy.nan, numpy.nan
4227 return numpy.nan, numpy.nan, numpy.nan
4219
4228
4220 def setup(self, dataOut, path, conf, samples, interval, mode):
4229 def setup(self, dataOut, path, conf, samples, interval, mode):
4221
4230
4222 self.path = path
4231 self.path = path
4223 self.conf = conf
4232 self.conf = conf
4224 self.samples = samples
4233 self.samples = samples
4225 self.interval = interval
4234 self.interval = interval
4226 self.mode = mode
4235 self.mode = mode
4227 if mode is None:
4236 if mode is None:
4228 self.flagAskMode = True
4237 self.flagAskMode = True
4229 N = 0
4238 N = 0
4230 while True:
4239 while True:
4231 if N == self.nTries+1:
4240 if N == self.nTries+1:
4232 log.error('No position files found in {}'.format(path), self.name)
4241 log.error('No position files found in {}'.format(path), self.name)
4233 raise IOError('No position files found in {}'.format(path))
4242 raise IOError('No position files found in {}'.format(path))
4234 filelist = self.find_file(dataOut.utctime)
4243 filelist = self.find_file(dataOut.utctime)
4235
4244
4236 if filelist == 0:
4245 if filelist == 0:
4237 N += 1
4246 N += 1
4238 log.warning('Waiting {}s for position files...'.format(self.delay), self.name)
4247 log.warning('Waiting {}s for position files...'.format(self.delay), self.name)
4239 time.sleep(self.delay)
4248 time.sleep(self.delay)
4240 continue
4249 continue
4241 self.filename = filelist[0]
4250 self.filename = filelist[0]
4242 try:
4251 try:
4243 self.fp = h5py.File(self.filename, 'r')
4252 self.fp = h5py.File(self.filename, 'r')
4244 self.utcfile = int(self.filename.split('/')[-1][4:14])
4253 self.utcfile = int(self.filename.split('/')[-1][4:14])
4254
4255 self.ele = self.fp['Data']['ele_pos'][:]
4256 self.azi = self.fp['Data']['azi_pos'][:] + 26.27
4257 self.azi[self.azi>360] = self.azi[self.azi>360] - 360
4245 break
4258 break
4246 except:
4259 except:
4247 log.warning('Waiting {}s for position file to be ready...'.format(self.delay), self.name)
4260 log.warning('Waiting {}s for position file to be ready...'.format(self.delay), self.name)
4248 time.sleep(self.delay)
4261 time.sleep(self.delay)
4249
4262
4250 def run(self, dataOut, path, conf=None, samples=1500, interval=0.04, time_offset=0, mode=None):
4263 def run(self, dataOut, path, conf=None, samples=1500, interval=0.04, time_offset=0, mode=None):
4251
4264
4252 if not self.isConfig:
4265 if not self.isConfig:
4253 self.setup(dataOut, path, conf, samples, interval, mode)
4266 self.setup(dataOut, path, conf, samples, interval, mode)
4254 self.isConfig = True
4267 self.isConfig = True
4255
4268
4256 self.utctime = dataOut.utctime + time_offset
4269 self.utctime = dataOut.utctime + time_offset
4257
4270
4258 self.find_next_file()
4271 self.find_next_file()
4259
4272
4260 az, el, scan = self.get_values()
4273 az, el, scan = self.get_values()
4274
4261 dataOut.flagNoData = False
4275 dataOut.flagNoData = False
4262 if numpy.isnan(az) or numpy.isnan(el) :
4276 if numpy.isnan(az) or numpy.isnan(el) :
4263 dataOut.flagNoData = True
4277 dataOut.flagNoData = True
4264 return dataOut
4278 return dataOut
4265
4279
4266 dataOut.azimuth = round(az, 2)
4280 dataOut.azimuth = round(az, 2)
4267 dataOut.elevation = round(el, 2)
4281 dataOut.elevation = round(el, 2)
4268 dataOut.mode_op = scan
4282 dataOut.mode_op = scan
4269
4283
4270 return dataOut
4284 return dataOut
4271
4285
4272
4286
4273 class Block360(Operation):
4287 class Block360(Operation):
4274 '''
4288 '''
4275 '''
4289 '''
4276 isConfig = False
4290 isConfig = False
4277 __profIndex = 0
4291 __profIndex = 0
4278 __initime = None
4292 __initime = None
4279 __lastdatatime = None
4293 __lastdatatime = None
4280 __buffer = None
4294 __buffer = None
4281 __dataReady = False
4295 __dataReady = False
4282 n = None
4296 n = None
4283 __nch = 0
4297 __nch = 0
4284 __nHeis = 0
4298 __nHeis = 0
4285 index = 0
4299 index = 0
4286 mode = None
4300 mode = None
4287
4301
4288 def __init__(self,**kwargs):
4302 def __init__(self,**kwargs):
4289 Operation.__init__(self,**kwargs)
4303 Operation.__init__(self,**kwargs)
4290
4304
4291 def setup(self, dataOut, attr):
4305 def setup(self, dataOut, attr):
4292 '''
4306 '''
4293 n= Numero de PRF's de entrada
4307 n= Numero de PRF's de entrada
4294 '''
4308 '''
4295 self.__initime = None
4309 self.__initime = None
4296 self.__lastdatatime = 0
4310 self.__lastdatatime = 0
4297 self.__dataReady = False
4311 self.__dataReady = False
4298 self.__buffer = 0
4312 self.__buffer = 0
4299 self.index = 0
4313 self.index = 0
4300 self.__nch = dataOut.nChannels
4314 self.__nch = dataOut.nChannels
4301 self.__nHeis = dataOut.nHeights
4315 self.__nHeis = dataOut.nHeights
4302
4316
4303 self.attr = attr
4317 self.attr = attr
4304
4318
4305 self.__buffer = []
4319 self.__buffer = []
4306 self.__buffer2 = []
4320 self.azi = []
4307 self.__buffer3 = []
4321 self.ele = []
4308 self.__buffer4 = []
4309
4322
4310 def putData(self, data, attr, flagMode):
4323 def putData(self, data, attr):
4311 '''
4324 '''
4312 Add a profile to he __buffer and increase in one the __profiel Index
4325 Add a profile to he __buffer and increase in one the __profiel Index
4313 '''
4326 '''
4314 tmp= getattr(data, attr)
4327 tmp= getattr(data, attr)
4315 self.__buffer.append(tmp)
4328 self.__buffer.append(tmp)
4316 self.__buffer2.append(data.azimuth)
4329 self.azi.append(data.azimuth)
4317 self.__buffer3.append(data.elevation)
4330 self.ele.append(data.elevation)
4318 self.__profIndex += 1
4331 self.__profIndex += 1
4319
4332
4320 if flagMode == 1: #'AZI'
4333 def pushData(self, data, case_flag):
4321 return numpy.array(self.__buffer2)
4322 elif flagMode == 0: #'ELE'
4323 return numpy.array(self.__buffer3)
4324
4325 def pushData(self, data,flagMode,case_flag):
4326 '''
4334 '''
4327 Return the PULSEPAIR and the profiles used in the operation
4328 Affected : self.__profileIndex
4329 '''
4335 '''
4330
4336
4331 data_360 = numpy.array(self.__buffer).transpose(1, 2, 0, 3)
4337 data_360 = numpy.array(self.__buffer).transpose(1, 2, 0, 3)
4332 data_p = numpy.array(self.__buffer2)
4338 data_p = numpy.array(self.azi)
4333 data_e = numpy.array(self.__buffer3)
4339 data_e = numpy.array(self.ele)
4334 n = self.__profIndex
4340 n = self.__profIndex
4335
4341
4336 self.__buffer = []
4342 self.__buffer = []
4337 self.__buffer2 = []
4343 self.azi = []
4338 self.__buffer3 = []
4344 self.ele = []
4339 self.__buffer4 = []
4340 self.__profIndex = 0
4345 self.__profIndex = 0
4341
4346
4342 if flagMode == 1 and case_flag == 0: #'AZI' y ha girado
4347 if case_flag in (0, 1, -1):
4343 self.putData(data=data, attr = self.attr, flagMode=flagMode)
4348 self.putData(data=data, attr = self.attr)
4344
4349
4345 return data_360, n, data_p, data_e
4350 return data_360, n, data_p, data_e
4346
4351
4347 def byProfiles(self,dataOut,flagMode):
4352 def byProfiles(self, dataOut):
4348
4353
4349 self.__dataReady = False
4354 self.__dataReady = False
4350 data_360 = []
4355 data_360 = []
4351 data_p = None
4356 data_p = None
4352 data_e = None
4357 data_e = None
4353
4358
4354 angles = self.putData(data=dataOut, attr = self.attr, flagMode=flagMode)
4359 self.putData(data=dataOut, attr = self.attr)
4355 if self.__profIndex > 1:
4360
4356 case_flag = self.checkcase(angles,flagMode)
4361 if self.__profIndex > 5:
4362 case_flag = self.checkcase()
4357
4363
4358 if flagMode == 1: #'AZI':
4364 if self.flagMode == 1: #'AZI':
4359 if case_flag == 0: #Ya giró
4365 if case_flag == 0: #Ya giró
4360 self.__buffer.pop() #Erase last data
4366 self.__buffer.pop() #Erase last data
4361 self.__buffer2.pop()
4367 self.azi.pop()
4362 self.__buffer3.pop()
4368 self.ele.pop()
4363 data_360 ,n,data_p,data_e = self.pushData(data=dataOut,flagMode=flagMode,case_flag=case_flag)
4369 data_360 ,n,data_p,data_e = self.pushData(dataOut, case_flag)
4364 self.__dataReady = True
4370 if len(data_p)>350:
4365
4366 elif flagMode == 0: #'ELE'
4367
4368 if case_flag == 0: #Subida
4369
4370 if len(self.__buffer) == 2: #Cuando está de subida
4371 #Se borra el dato anterior para liberar buffer y comparar el dato actual con el siguiente
4372 self.__buffer.pop(0) #Erase first data
4373 self.__buffer2.pop(0)
4374 self.__buffer3.pop(0)
4375 self.__profIndex -= 1
4376 else: #Cuando ha estado de bajada y ha vuelto a subir
4377 #Se borra el último dato
4378 self.__buffer.pop() #Erase last data
4379 self.__buffer2.pop()
4380 self.__buffer3.pop()
4381 data_360, n, data_p, data_e = self.pushData(data=dataOut,flagMode=flagMode,case_flag=case_flag)
4382 self.__dataReady = True
4371 self.__dataReady = True
4372 elif self.flagMode == 0: #'ELE'
4373 if case_flag == 1: #Bajada
4374 self.__buffer.pop() #Erase last data
4375 self.azi.pop()
4376 self.ele.pop()
4377 data_360, n, data_p, data_e = self.pushData(dataOut, case_flag)
4378 self.__dataReady = True
4379 if case_flag == -1: #Subida
4380 self.__buffer.pop() #Erase last data
4381 self.azi.pop()
4382 self.ele.pop()
4383 data_360, n, data_p, data_e = self.pushData(dataOut, case_flag)
4384 #self.__dataReady = True
4383
4385
4384 return data_360, data_p, data_e
4386 return data_360, data_p, data_e
4385
4387
4386
4388
4387 def blockOp(self, dataOut, flagMode, datatime= None):
4389 def blockOp(self, dataOut, datatime= None):
4388 if self.__initime == None:
4390 if self.__initime == None:
4389 self.__initime = datatime
4391 self.__initime = datatime
4390 data_360, data_p, data_e = self.byProfiles(dataOut,flagMode)
4392 data_360, data_p, data_e = self.byProfiles(dataOut)
4391 self.__lastdatatime = datatime
4393 self.__lastdatatime = datatime
4392
4394
4393 avgdatatime = self.__initime
4395 avgdatatime = self.__initime
4394 if self.n==1:
4396 if self.n==1:
4395 avgdatatime = datatime
4397 avgdatatime = datatime
4396 deltatime = datatime - self.__lastdatatime
4398
4397 self.__initime = datatime
4399 self.__initime = datatime
4398 return data_360, avgdatatime, data_p, data_e
4400 return data_360, avgdatatime, data_p, data_e
4399
4401
4400 def checkcase(self, angles, flagMode):
4402 def checkcase(self):
4403
4404 sigma_ele = numpy.nanstd(self.ele[-5:])
4405 sigma_azi = numpy.nanstd(self.azi[-5:])
4401
4406
4402 if flagMode == 1: #'AZI'
4407 if sigma_ele<.5 and sigma_azi<.5:
4403 start = angles[-2]
4408 if sigma_ele<sigma_azi:
4404 end = angles[-1]
4409 self.flagMode = 1
4410 self.mode_op = 'PPI'
4411 else:
4412 self.flagMode = 0
4413 self.mode_op = 'RHI'
4414 elif sigma_ele < .5:
4415 self.flagMode = 1
4416 self.mode_op = 'PPI'
4417 elif sigma_azi < .5:
4418 self.flagMode = 0
4419 self.mode_op = 'RHI'
4420 else:
4421 self.flagMode = None
4422 self.mode_op = 'None'
4423
4424 if self.flagMode == 1: #'AZI'
4425 start = self.azi[-2]
4426 end = self.azi[-1]
4405 diff_angle = (end-start)
4427 diff_angle = (end-start)
4406
4428
4407 if diff_angle < 0: #Ya giró
4429 if diff_angle < 0: #Ya giró
4408 return 0
4430 return 0
4409
4431
4410 elif flagMode == 0: #'ELE'
4432 elif self.flagMode == 0: #'ELE'
4411
4433
4412 start = angles[-2]
4434 start = self.ele[-3]
4413 end = angles[-1]
4435 middle = self.ele[-2]
4414 diff_angle = (end-start)
4436 end = self.ele[-1]
4415
4437
4416 if diff_angle > 0: #Subida
4438 if end < 0:
4417 return 0
4439 return 1
4440 elif (middle>start and end<middle):
4441 return -1
4418
4442
4419 def run(self, dataOut, attr_data='dataPP_POWER', runNextOp = False,**kwargs):
4443 def run(self, dataOut, attr_data='dataPP_POWER', runNextOp = False,**kwargs):
4420
4444
4421 dataOut.attr_data = attr_data
4445 dataOut.attr_data = attr_data
4422 dataOut.runNextOp = runNextOp
4446 dataOut.runNextOp = runNextOp
4423 dataOut.flagAskMode = False
4424
4425 if dataOut.mode_op == 'PPI':
4426 dataOut.flagMode = 1
4427 elif dataOut.mode_op == 'RHI':
4428 dataOut.flagMode = 0
4429
4447
4430 if not self.isConfig:
4448 if not self.isConfig:
4431 self.setup(dataOut = dataOut, attr = attr_data ,**kwargs)
4449 self.setup(dataOut = dataOut, attr = attr_data ,**kwargs)
4432 self.isConfig = True
4450 self.isConfig = True
4433
4451
4434 data_360, avgdatatime, data_p, data_e = self.blockOp(dataOut, dataOut.flagMode, dataOut.utctime)
4452 data_360, avgdatatime, data_p, data_e = self.blockOp(dataOut, dataOut.utctime)
4435
4453
4436 dataOut.flagNoData = True
4454 dataOut.flagNoData = True
4437
4455
4438 if self.__dataReady:
4456 if self.__dataReady:
4439 setattr(dataOut, attr_data, data_360 )
4457 setattr(dataOut, attr_data, data_360 )
4440 dataOut.data_azi = data_p + 26.2
4458 dataOut.data_azi = data_p
4441 dataOut.data_azi[dataOut.data_azi>360] = dataOut.data_azi[dataOut.data_azi>360] - 360
4442 dataOut.data_ele = data_e
4459 dataOut.data_ele = data_e
4443 dataOut.utctime = avgdatatime
4460 dataOut.utctime = avgdatatime
4444 dataOut.flagNoData = False
4461 dataOut.flagNoData = False
4445 dataOut.flagAskMode = True
4462 dataOut.flagMode = self.flagMode
4463 dataOut.mode_op = self.mode_op
4446
4464
4447 return dataOut
4465 return dataOut
4448
4466
4449 class MergeProc(ProcessingUnit):
4467 class MergeProc(ProcessingUnit):
4450
4468
4451 def __init__(self):
4469 def __init__(self):
4452 ProcessingUnit.__init__(self)
4470 ProcessingUnit.__init__(self)
4453
4471
4454 def run(self, attr_data, mode=0):
4472 def run(self, attr_data, mode=0):
4455
4473
4456 #exit(1)
4474 #exit(1)
4457 self.dataOut = getattr(self, self.inputs[0])
4475 self.dataOut = getattr(self, self.inputs[0])
4458 data_inputs = [getattr(self, attr) for attr in self.inputs]
4476 data_inputs = [getattr(self, attr) for attr in self.inputs]
4459 #print(data_inputs)
4477 #print(data_inputs)
4460 #print(numpy.shape([getattr(data, attr_data) for data in data_inputs][1]))
4478 #print(numpy.shape([getattr(data, attr_data) for data in data_inputs][1]))
4461 #exit(1)
4479 #exit(1)
4462 if mode==0:
4480 if mode==0:
4463 data = numpy.concatenate([getattr(data, attr_data) for data in data_inputs])
4481 data = numpy.concatenate([getattr(data, attr_data) for data in data_inputs])
4464 setattr(self.dataOut, attr_data, data)
4482 setattr(self.dataOut, attr_data, data)
4465
4483
4466 if mode==1: #Hybrid
4484 if mode==1: #Hybrid
4467 #data = numpy.concatenate([getattr(data, attr_data) for data in data_inputs],axis=1)
4485 #data = numpy.concatenate([getattr(data, attr_data) for data in data_inputs],axis=1)
4468 #setattr(self.dataOut, attr_data, data)
4486 #setattr(self.dataOut, attr_data, data)
4469 setattr(self.dataOut, 'dataLag_spc', [getattr(data, attr_data) for data in data_inputs][0])
4487 setattr(self.dataOut, 'dataLag_spc', [getattr(data, attr_data) for data in data_inputs][0])
4470 setattr(self.dataOut, 'dataLag_spc_LP', [getattr(data, attr_data) for data in data_inputs][1])
4488 setattr(self.dataOut, 'dataLag_spc_LP', [getattr(data, attr_data) for data in data_inputs][1])
4471 setattr(self.dataOut, 'dataLag_cspc', [getattr(data, attr_data_2) for data in data_inputs][0])
4489 setattr(self.dataOut, 'dataLag_cspc', [getattr(data, attr_data_2) for data in data_inputs][0])
4472 setattr(self.dataOut, 'dataLag_cspc_LP', [getattr(data, attr_data_2) for data in data_inputs][1])
4490 setattr(self.dataOut, 'dataLag_cspc_LP', [getattr(data, attr_data_2) for data in data_inputs][1])
4473 #setattr(self.dataOut, 'nIncohInt', [getattr(data, attr_data_3) for data in data_inputs][0])
4491 #setattr(self.dataOut, 'nIncohInt', [getattr(data, attr_data_3) for data in data_inputs][0])
4474 #setattr(self.dataOut, 'nIncohInt_LP', [getattr(data, attr_data_3) for data in data_inputs][1])
4492 #setattr(self.dataOut, 'nIncohInt_LP', [getattr(data, attr_data_3) for data in data_inputs][1])
4475 '''
4493 '''
4476 print(self.dataOut.dataLag_spc_LP.shape)
4494 print(self.dataOut.dataLag_spc_LP.shape)
4477 print(self.dataOut.dataLag_cspc_LP.shape)
4495 print(self.dataOut.dataLag_cspc_LP.shape)
4478 exit(1)
4496 exit(1)
4479 '''
4497 '''
4480
4498
4481 #self.dataOut.dataLag_spc_LP = numpy.transpose(self.dataOut.dataLag_spc_LP[0],(2,0,1))
4499 #self.dataOut.dataLag_spc_LP = numpy.transpose(self.dataOut.dataLag_spc_LP[0],(2,0,1))
4482 #self.dataOut.dataLag_cspc_LP = numpy.transpose(self.dataOut.dataLag_cspc_LP,(3,1,2,0))
4500 #self.dataOut.dataLag_cspc_LP = numpy.transpose(self.dataOut.dataLag_cspc_LP,(3,1,2,0))
4483 '''
4501 '''
4484 print("Merge")
4502 print("Merge")
4485 print(numpy.shape(self.dataOut.dataLag_spc))
4503 print(numpy.shape(self.dataOut.dataLag_spc))
4486 print(numpy.shape(self.dataOut.dataLag_spc_LP))
4504 print(numpy.shape(self.dataOut.dataLag_spc_LP))
4487 print(numpy.shape(self.dataOut.dataLag_cspc))
4505 print(numpy.shape(self.dataOut.dataLag_cspc))
4488 print(numpy.shape(self.dataOut.dataLag_cspc_LP))
4506 print(numpy.shape(self.dataOut.dataLag_cspc_LP))
4489 exit(1)
4507 exit(1)
4490 '''
4508 '''
4491 #print(numpy.sum(self.dataOut.dataLag_spc_LP[2,:,164])/128)
4509 #print(numpy.sum(self.dataOut.dataLag_spc_LP[2,:,164])/128)
4492 #print(numpy.sum(self.dataOut.dataLag_cspc_LP[0,:,30,1])/128)
4510 #print(numpy.sum(self.dataOut.dataLag_cspc_LP[0,:,30,1])/128)
4493 #exit(1)
4511 #exit(1)
4494 #print(self.dataOut.NDP)
4512 #print(self.dataOut.NDP)
4495 #print(self.dataOut.nNoiseProfiles)
4513 #print(self.dataOut.nNoiseProfiles)
4496
4514
4497 #self.dataOut.nIncohInt_LP = 128
4515 #self.dataOut.nIncohInt_LP = 128
4498 self.dataOut.nProfiles_LP = 128#self.dataOut.nIncohInt_LP
4516 self.dataOut.nProfiles_LP = 128#self.dataOut.nIncohInt_LP
4499 self.dataOut.nIncohInt_LP = self.dataOut.nIncohInt
4517 self.dataOut.nIncohInt_LP = self.dataOut.nIncohInt
4500 self.dataOut.NLAG = 16
4518 self.dataOut.NLAG = 16
4501 self.dataOut.NRANGE = 200
4519 self.dataOut.NRANGE = 200
4502 self.dataOut.NSCAN = 128
4520 self.dataOut.NSCAN = 128
4503 #print(numpy.shape(self.dataOut.data_spc))
4521 #print(numpy.shape(self.dataOut.data_spc))
4504
4522
4505 #exit(1)
4523 #exit(1)
4506
4524
4507 if mode==2: #HAE 2022
4525 if mode==2: #HAE 2022
4508 data = numpy.sum([getattr(data, attr_data) for data in data_inputs],axis=0)
4526 data = numpy.sum([getattr(data, attr_data) for data in data_inputs],axis=0)
4509 setattr(self.dataOut, attr_data, data)
4527 setattr(self.dataOut, attr_data, data)
4510
4528
4511 self.dataOut.nIncohInt *= 2
4529 self.dataOut.nIncohInt *= 2
4512 #meta = self.dataOut.getFreqRange(1)/1000.
4530 #meta = self.dataOut.getFreqRange(1)/1000.
4513 self.dataOut.freqRange = self.dataOut.getFreqRange(1)/1000.
4531 self.dataOut.freqRange = self.dataOut.getFreqRange(1)/1000.
4514
4532
4515 #exit(1)
4533 #exit(1)
4516
4534
4517 if mode==7: #RM
4535 if mode==7: #RM
4518
4536
4519 f = [getattr(data, attr_data) for data in data_inputs][0]
4537 f = [getattr(data, attr_data) for data in data_inputs][0]
4520 g = [getattr(data, attr_data) for data in data_inputs][1]
4538 g = [getattr(data, attr_data) for data in data_inputs][1]
4521 data = numpy.concatenate((f,g),axis=3)
4539 data = numpy.concatenate((f,g),axis=3)
4522 setattr(self.dataOut, attr_data, data)
4540 setattr(self.dataOut, attr_data, data)
4523
4541
4524 # snr
4542 # snr
4525 # self.dataOut.data_snr = numpy.concatenate((data_inputs[0].data_snr, data_inputs[1].data_snr), axis=2)
4543 # self.dataOut.data_snr = numpy.concatenate((data_inputs[0].data_snr, data_inputs[1].data_snr), axis=2)
4526
4544
4527 # ranges
4545 # ranges
4528 dh = self.dataOut.heightList[1]-self.dataOut.heightList[0]
4546 dh = self.dataOut.heightList[1]-self.dataOut.heightList[0]
4529 heightList_2 = (self.dataOut.heightList[-1]+dh) + numpy.arange(g.shape[-1], dtype=numpy.float) * dh
4547 heightList_2 = (self.dataOut.heightList[-1]+dh) + numpy.arange(g.shape[-1], dtype=numpy.float) * dh
4530
4548
4531 self.dataOut.heightList = numpy.concatenate((self.dataOut.heightList,heightList_2))
4549 self.dataOut.heightList = numpy.concatenate((self.dataOut.heightList,heightList_2))
General Comments 0
You need to be logged in to leave comments. Login now